VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 50867

Last change on this file since 50867 was 50867, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Added hmR0VmxSetPendingXcptGP() and some doxygen fixes.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 479.1 KB
Line 
1/* $Id: HMVMXR0.cpp 50867 2014-03-25 15:47:45Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_SAVE_FULL_GUEST_STATE
41# define HMVMX_SYNC_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44# define HMVMX_ALWAYS_TRAP_PF
45# define HMVMX_ALWAYS_SWAP_FPU_STATE
46# define HMVMX_ALWAYS_FLUSH_TLB
47#endif
48
49
50/*******************************************************************************
51* Defined Constants And Macros *
52*******************************************************************************/
53#if defined(RT_ARCH_AMD64)
54# define HMVMX_IS_64BIT_HOST_MODE() (true)
55typedef RTHCUINTREG HMVMXHCUINTREG;
56#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57extern "C" uint32_t g_fVMXIs64bitHost;
58# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59typedef uint64_t HMVMXHCUINTREG;
60#else
61# define HMVMX_IS_64BIT_HOST_MODE() (false)
62typedef RTHCUINTREG HMVMXHCUINTREG;
63#endif
64
65/** Use the function table. */
66#define HMVMX_USE_FUNCTION_TABLE
67
68/** Determine which tagged-TLB flush handler to use. */
69#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74/** @name Updated-guest-state flags.
75 * @{ */
76#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96 | HMVMX_UPDATED_GUEST_RSP \
97 | HMVMX_UPDATED_GUEST_RFLAGS \
98 | HMVMX_UPDATED_GUEST_CR0 \
99 | HMVMX_UPDATED_GUEST_CR3 \
100 | HMVMX_UPDATED_GUEST_CR4 \
101 | HMVMX_UPDATED_GUEST_GDTR \
102 | HMVMX_UPDATED_GUEST_IDTR \
103 | HMVMX_UPDATED_GUEST_LDTR \
104 | HMVMX_UPDATED_GUEST_TR \
105 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106 | HMVMX_UPDATED_GUEST_DEBUG \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
112 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113 | HMVMX_UPDATED_GUEST_APIC_STATE)
114/** @} */
115
116/** @name
117 * Flags to skip redundant reads of some common VMCS fields that are not part of
118 * the guest-CPU state but are in the transient structure.
119 */
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127/** @} */
128
129/** @name
130 * States of the VMCS.
131 *
132 * This does not reflect all possible VMCS states but currently only those
133 * needed for maintaining the VMCS consistently even when thread-context hooks
134 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
135 */
136#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139/** @} */
140
141/**
142 * Exception bitmap mask for real-mode guests (real-on-v86).
143 *
144 * We need to intercept all exceptions manually (except #PF). #NM is also
145 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146 * even in real-mode if we have Nested Paging support.
147 */
148#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
149 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
150 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
151 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
152 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
154 | RT_BIT(X86_XCPT_XF))
155
156/**
157 * Exception bitmap mask for all contributory exceptions.
158 *
159 * Page fault is deliberately excluded here as it's conditional as to whether
160 * it's contributory or benign. Page faults are handled separately.
161 */
162#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
163 | RT_BIT(X86_XCPT_DE))
164
165/** Maximum VM-instruction error number. */
166#define HMVMX_INSTR_ERROR_MAX 28
167
168/** Profiling macro. */
169#ifdef HM_PROFILE_EXIT_DISPATCH
170# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172#else
173# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175#endif
176
177/** Assert that preemption is disabled or covered by thread-context hooks. */
178#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181/** Assert that we haven't migrated CPUs when thread-context hooks are not
182 * used. */
183#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185 ("Illegal migration! Entered on CPU %u Current %u\n", \
186 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188/** Helper macro for VM-exit handlers called unexpectedly. */
189#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190 do { \
191 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192 return VERR_VMX_UNEXPECTED_EXIT; \
193 } while (0)
194
195
196/*******************************************************************************
197* Structures and Typedefs *
198*******************************************************************************/
199/**
200 * VMX transient state.
201 *
202 * A state structure for holding miscellaneous information across
203 * VMX non-root operation and restored after the transition.
204 */
205typedef struct VMXTRANSIENT
206{
207 /** The host's rflags/eflags. */
208 RTCCUINTREG uEflags;
209#if HC_ARCH_BITS == 32
210 uint32_t u32Alignment0;
211#endif
212 /** The guest's TPR value used for TPR shadowing. */
213 uint8_t u8GuestTpr;
214 /** Alignment. */
215 uint8_t abAlignment0[7];
216
217 /** The basic VM-exit reason. */
218 uint16_t uExitReason;
219 /** Alignment. */
220 uint16_t u16Alignment0;
221 /** The VM-exit interruption error code. */
222 uint32_t uExitIntErrorCode;
223 /** The VM-exit exit qualification. */
224 uint64_t uExitQualification;
225
226 /** The VM-exit interruption-information field. */
227 uint32_t uExitIntInfo;
228 /** The VM-exit instruction-length field. */
229 uint32_t cbInstr;
230 /** The VM-exit instruction-information field. */
231 union
232 {
233 /** Plain unsigned int representation. */
234 uint32_t u;
235 /** INS and OUTS information. */
236 struct
237 {
238 uint32_t u6Reserved0 : 7;
239 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240 uint32_t u3AddrSize : 3;
241 uint32_t u5Reserved1 : 5;
242 /** The segment register (X86_SREG_XXX). */
243 uint32_t iSegReg : 3;
244 uint32_t uReserved2 : 14;
245 } StrIo;
246 } ExitInstrInfo;
247 /** Whether the VM-entry failed or not. */
248 bool fVMEntryFailed;
249 /** Alignment. */
250 uint8_t abAlignment1[3];
251
252 /** The VM-entry interruption-information field. */
253 uint32_t uEntryIntInfo;
254 /** The VM-entry exception error code field. */
255 uint32_t uEntryXcptErrorCode;
256 /** The VM-entry instruction length field. */
257 uint32_t cbEntryInstr;
258
259 /** IDT-vectoring information field. */
260 uint32_t uIdtVectoringInfo;
261 /** IDT-vectoring error code. */
262 uint32_t uIdtVectoringErrorCode;
263
264 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
265 uint32_t fVmcsFieldsRead;
266
267 /** Whether the guest FPU was active at the time of VM-exit. */
268 bool fWasGuestFPUStateActive;
269 /** Whether the guest debug state was active at the time of VM-exit. */
270 bool fWasGuestDebugStateActive;
271 /** Whether the hyper debug state was active at the time of VM-exit. */
272 bool fWasHyperDebugStateActive;
273 /** Whether TSC-offsetting should be setup before VM-entry. */
274 bool fUpdateTscOffsettingAndPreemptTimer;
275 /** Whether the VM-exit was caused by a page-fault during delivery of a
276 * contributory exception or a page-fault. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns VBox status code.
319 * @param pVCpu Pointer to the VMCPU.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333
334/*******************************************************************************
335* Internal Functions *
336*******************************************************************************/
337static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343#endif
344#ifndef HMVMX_USE_FUNCTION_TABLE
345DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346# define HMVMX_EXIT_DECL static int
347#else
348# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349#endif
350
351/** @name VM-exit handlers.
352 * @{
353 */
354static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358static FNVMXEXITHANDLER hmR0VmxExitSipi;
359static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360static FNVMXEXITHANDLER hmR0VmxExitSmi;
361static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366static FNVMXEXITHANDLER hmR0VmxExitHlt;
367static FNVMXEXITHANDLER hmR0VmxExitInvd;
368static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371static FNVMXEXITHANDLER hmR0VmxExitRsm;
372static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381static FNVMXEXITHANDLER hmR0VmxExitMwait;
382static FNVMXEXITHANDLER hmR0VmxExitMtf;
383static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384static FNVMXEXITHANDLER hmR0VmxExitPause;
385static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398/** @} */
399
400static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408#endif
409static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411/*******************************************************************************
412* Global Variables *
413*******************************************************************************/
414#ifdef HMVMX_USE_FUNCTION_TABLE
415
416/**
417 * VMX_EXIT dispatch table.
418 */
419static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420{
421 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461 /* 40 UNDEFINED */ hmR0VmxExitPause,
462 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481};
482#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484#ifdef VBOX_STRICT
485static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486{
487 /* 0 */ "(Not Used)",
488 /* 1 */ "VMCALL executed in VMX root operation.",
489 /* 2 */ "VMCLEAR with invalid physical address.",
490 /* 3 */ "VMCLEAR with VMXON pointer.",
491 /* 4 */ "VMLAUNCH with non-clear VMCS.",
492 /* 5 */ "VMRESUME with non-launched VMCS.",
493 /* 6 */ "VMRESUME after VMXOFF",
494 /* 7 */ "VM entry with invalid control fields.",
495 /* 8 */ "VM entry with invalid host state fields.",
496 /* 9 */ "VMPTRLD with invalid physical address.",
497 /* 10 */ "VMPTRLD with VMXON pointer.",
498 /* 11 */ "VMPTRLD with incorrect revision identifier.",
499 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500 /* 13 */ "VMWRITE to read-only VMCS component.",
501 /* 14 */ "(Not Used)",
502 /* 15 */ "VMXON executed in VMX root operation.",
503 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
504 /* 17 */ "VM entry with non-launched executing VMCS.",
505 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506 /* 19 */ "VMCALL with non-clear VMCS.",
507 /* 20 */ "VMCALL with invalid VM-exit control fields.",
508 /* 21 */ "(Not Used)",
509 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511 /* 24 */ "VMCALL with invalid SMM-monitor features.",
512 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513 /* 26 */ "VM entry with events blocked by MOV SS.",
514 /* 27 */ "(Not Used)",
515 /* 28 */ "Invalid operand to INVEPT/INVVPID."
516};
517#endif /* VBOX_STRICT */
518
519
520
521/**
522 * Updates the VM's last error record. If there was a VMX instruction error,
523 * reads the error data from the VMCS and updates VCPU's last error record as
524 * well.
525 *
526 * @param pVM Pointer to the VM.
527 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528 * VERR_VMX_UNABLE_TO_START_VM or
529 * VERR_VMX_INVALID_VMCS_FIELD).
530 * @param rc The error code.
531 */
532static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533{
534 AssertPtr(pVM);
535 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536 || rc == VERR_VMX_UNABLE_TO_START_VM)
537 {
538 AssertPtrReturnVoid(pVCpu);
539 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540 }
541 pVM->hm.s.lLastError = rc;
542}
543
544
545/**
546 * Reads the VM-entry interruption-information field from the VMCS into the VMX
547 * transient structure.
548 *
549 * @returns VBox status code.
550 * @param pVmxTransient Pointer to the VMX transient structure.
551 *
552 * @remarks No-long-jump zone!!!
553 */
554DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555{
556 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557 AssertRCReturn(rc, rc);
558 return VINF_SUCCESS;
559}
560
561
562/**
563 * Reads the VM-entry exception error code field from the VMCS into
564 * the VMX transient structure.
565 *
566 * @returns VBox status code.
567 * @param pVmxTransient Pointer to the VMX transient structure.
568 *
569 * @remarks No-long-jump zone!!!
570 */
571DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572{
573 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574 AssertRCReturn(rc, rc);
575 return VINF_SUCCESS;
576}
577
578
579/**
580 * Reads the VM-entry exception error code field from the VMCS into
581 * the VMX transient structure.
582 *
583 * @returns VBox status code.
584 * @param pVmxTransient Pointer to the VMX transient structure.
585 *
586 * @remarks No-long-jump zone!!!
587 */
588DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589{
590 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591 AssertRCReturn(rc, rc);
592 return VINF_SUCCESS;
593}
594
595
596/**
597 * Reads the VM-exit interruption-information field from the VMCS into the VMX
598 * transient structure.
599 *
600 * @returns VBox status code.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 */
603DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606 {
607 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608 AssertRCReturn(rc, rc);
609 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610 }
611 return VINF_SUCCESS;
612}
613
614
615/**
616 * Reads the VM-exit interruption error code from the VMCS into the VMX
617 * transient structure.
618 *
619 * @returns VBox status code.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction length field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu Pointer to the VMCPU.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the VM-exit instruction-information field from the VMCS into
656 * the VMX transient structure.
657 *
658 * @returns VBox status code.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664 {
665 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the exit qualification from the VMCS into the VMX transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678 * case).
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684 {
685 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the IDT-vectoring information field from the VMCS into the VMX
695 * transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 *
700 * @remarks No-long-jump zone!!!
701 */
702DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703{
704 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705 {
706 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707 AssertRCReturn(rc, rc);
708 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709 }
710 return VINF_SUCCESS;
711}
712
713
714/**
715 * Reads the IDT-vectoring error code from the VMCS into the VMX
716 * transient structure.
717 *
718 * @returns VBox status code.
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724 {
725 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Enters VMX root mode operation on the current CPU.
735 *
736 * @returns VBox status code.
737 * @param pVM Pointer to the VM (optional, can be NULL, after
738 * a resume).
739 * @param HCPhysCpuPage Physical address of the VMXON region.
740 * @param pvCpuPage Pointer to the VMXON region.
741 */
742static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743{
744 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
745 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
746 Assert(pvCpuPage);
747 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
748
749 if (pVM)
750 {
751 /* Write the VMCS revision dword to the VMXON region. */
752 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
753 }
754
755 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
756 RTCCUINTREG uEflags = ASMIntDisableFlags();
757
758 /* Enable the VMX bit in CR4 if necessary. */
759 RTCCUINTREG uCr4 = ASMGetCR4();
760 if (!(uCr4 & X86_CR4_VMXE))
761 ASMSetCR4(uCr4 | X86_CR4_VMXE);
762
763 /* Enter VMX root mode. */
764 int rc = VMXEnable(HCPhysCpuPage);
765 if (RT_FAILURE(rc))
766 ASMSetCR4(uCr4);
767
768 /* Restore interrupts. */
769 ASMSetFlags(uEflags);
770 return rc;
771}
772
773
774/**
775 * Exits VMX root mode operation on the current CPU.
776 *
777 * @returns VBox status code.
778 */
779static int hmR0VmxLeaveRootMode(void)
780{
781 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
782
783 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
784 RTCCUINTREG uEflags = ASMIntDisableFlags();
785
786 /* If we're for some reason not in VMX root mode, then don't leave it. */
787 RTCCUINTREG uHostCR4 = ASMGetCR4();
788
789 int rc;
790 if (uHostCR4 & X86_CR4_VMXE)
791 {
792 /* Exit VMX root mode and clear the VMX bit in CR4. */
793 VMXDisable();
794 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
795 rc = VINF_SUCCESS;
796 }
797 else
798 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
799
800 /* Restore interrupts. */
801 ASMSetFlags(uEflags);
802 return rc;
803}
804
805
806/**
807 * Allocates and maps one physically contiguous page. The allocated page is
808 * zero'd out. (Used by various VT-x structures).
809 *
810 * @returns IPRT status code.
811 * @param pMemObj Pointer to the ring-0 memory object.
812 * @param ppVirt Where to store the virtual address of the
813 * allocation.
814 * @param pPhys Where to store the physical address of the
815 * allocation.
816 */
817DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
818{
819 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
820 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
821 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
822
823 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
824 if (RT_FAILURE(rc))
825 return rc;
826 *ppVirt = RTR0MemObjAddress(*pMemObj);
827 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
828 ASMMemZero32(*ppVirt, PAGE_SIZE);
829 return VINF_SUCCESS;
830}
831
832
833/**
834 * Frees and unmaps an allocated physical page.
835 *
836 * @param pMemObj Pointer to the ring-0 memory object.
837 * @param ppVirt Where to re-initialize the virtual address of
838 * allocation as 0.
839 * @param pHCPhys Where to re-initialize the physical address of the
840 * allocation as 0.
841 */
842DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
843{
844 AssertPtr(pMemObj);
845 AssertPtr(ppVirt);
846 AssertPtr(pHCPhys);
847 if (*pMemObj != NIL_RTR0MEMOBJ)
848 {
849 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
850 AssertRC(rc);
851 *pMemObj = NIL_RTR0MEMOBJ;
852 *ppVirt = 0;
853 *pHCPhys = 0;
854 }
855}
856
857
858/**
859 * Worker function to free VT-x related structures.
860 *
861 * @returns IPRT status code.
862 * @param pVM Pointer to the VM.
863 */
864static void hmR0VmxStructsFree(PVM pVM)
865{
866 for (VMCPUID i = 0; i < pVM->cCpus; i++)
867 {
868 PVMCPU pVCpu = &pVM->aCpus[i];
869 AssertPtr(pVCpu);
870
871 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
872 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873
874 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879 }
880
881 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884#endif
885}
886
887
888/**
889 * Worker function to allocate VT-x related VM structures.
890 *
891 * @returns IPRT status code.
892 * @param pVM Pointer to the VM.
893 */
894static int hmR0VmxStructsAlloc(PVM pVM)
895{
896 /*
897 * Initialize members up-front so we can cleanup properly on allocation failure.
898 */
899#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902 pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911#endif
912 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915 for (VMCPUID i = 0; i < pVM->cCpus; i++)
916 {
917 PVMCPU pVCpu = &pVM->aCpus[i];
918 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
923 }
924#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
925#undef VMXLOCAL_INIT_VM_MEMOBJ
926
927 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
928 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
929 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
930 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
931
932 /*
933 * Allocate all the VT-x structures.
934 */
935 int rc = VINF_SUCCESS;
936#ifdef VBOX_WITH_CRASHDUMP_MAGIC
937 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
938 if (RT_FAILURE(rc))
939 goto cleanup;
940 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
941 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
942#endif
943
944 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
945 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
946 {
947 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
948 &pVM->hm.s.vmx.HCPhysApicAccess);
949 if (RT_FAILURE(rc))
950 goto cleanup;
951 }
952
953 /*
954 * Initialize per-VCPU VT-x structures.
955 */
956 for (VMCPUID i = 0; i < pVM->cCpus; i++)
957 {
958 PVMCPU pVCpu = &pVM->aCpus[i];
959 AssertPtr(pVCpu);
960
961 /* Allocate the VM control structure (VMCS). */
962 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
963 if (RT_FAILURE(rc))
964 goto cleanup;
965
966 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
967 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
968 {
969 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
970 &pVCpu->hm.s.vmx.HCPhysVirtApic);
971 if (RT_FAILURE(rc))
972 goto cleanup;
973 }
974
975 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
976 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
977 {
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
979 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
983 }
984
985 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
986 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989
990 /* Allocate the VM-exit MSR-load page for the host MSRs. */
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
992 if (RT_FAILURE(rc))
993 goto cleanup;
994 }
995
996 return VINF_SUCCESS;
997
998cleanup:
999 hmR0VmxStructsFree(pVM);
1000 return rc;
1001}
1002
1003
1004/**
1005 * Does global VT-x initialization (called during module initialization).
1006 *
1007 * @returns VBox status code.
1008 */
1009VMMR0DECL(int) VMXR0GlobalInit(void)
1010{
1011#ifdef HMVMX_USE_FUNCTION_TABLE
1012 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1013# ifdef VBOX_STRICT
1014 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1015 Assert(g_apfnVMExitHandlers[i]);
1016# endif
1017#endif
1018 return VINF_SUCCESS;
1019}
1020
1021
1022/**
1023 * Does global VT-x termination (called during module termination).
1024 */
1025VMMR0DECL(void) VMXR0GlobalTerm()
1026{
1027 /* Nothing to do currently. */
1028}
1029
1030
1031/**
1032 * Sets up and activates VT-x on the current CPU.
1033 *
1034 * @returns VBox status code.
1035 * @param pCpu Pointer to the global CPU info struct.
1036 * @param pVM Pointer to the VM (can be NULL after a host resume
1037 * operation).
1038 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1039 * fEnabledByHost is true).
1040 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1041 * @a fEnabledByHost is true).
1042 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1043 * enable VT-x on the host.
1044 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1045 */
1046VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1047 void *pvMsrs)
1048{
1049 Assert(pCpu);
1050 Assert(pvMsrs);
1051 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1052
1053 /* Enable VT-x if it's not already enabled by the host. */
1054 if (!fEnabledByHost)
1055 {
1056 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1057 if (RT_FAILURE(rc))
1058 return rc;
1059 }
1060
1061 /*
1062 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1063 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1064 */
1065 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1066 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1067 {
1068 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1069 pCpu->fFlushAsidBeforeUse = false;
1070 }
1071 else
1072 pCpu->fFlushAsidBeforeUse = true;
1073
1074 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1075 ++pCpu->cTlbFlushes;
1076
1077 return VINF_SUCCESS;
1078}
1079
1080
1081/**
1082 * Deactivates VT-x on the current CPU.
1083 *
1084 * @returns VBox status code.
1085 * @param pCpu Pointer to the global CPU info struct.
1086 * @param pvCpuPage Pointer to the VMXON region.
1087 * @param HCPhysCpuPage Physical address of the VMXON region.
1088 *
1089 * @remarks This function should never be called when SUPR0EnableVTx() or
1090 * similar was used to enable VT-x on the host.
1091 */
1092VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1093{
1094 NOREF(pCpu);
1095 NOREF(pvCpuPage);
1096 NOREF(HCPhysCpuPage);
1097
1098 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1099 return hmR0VmxLeaveRootMode();
1100}
1101
1102
1103/**
1104 * Sets the permission bits for the specified MSR in the MSR bitmap.
1105 *
1106 * @param pVCpu Pointer to the VMCPU.
1107 * @param uMSR The MSR value.
1108 * @param enmRead Whether reading this MSR causes a VM-exit.
1109 * @param enmWrite Whether writing this MSR causes a VM-exit.
1110 */
1111static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1112{
1113 int32_t iBit;
1114 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1115
1116 /*
1117 * Layout:
1118 * 0x000 - 0x3ff - Low MSR read bits
1119 * 0x400 - 0x7ff - High MSR read bits
1120 * 0x800 - 0xbff - Low MSR write bits
1121 * 0xc00 - 0xfff - High MSR write bits
1122 */
1123 if (uMsr <= 0x00001FFF)
1124 iBit = uMsr;
1125 else if ( uMsr >= 0xC0000000
1126 && uMsr <= 0xC0001FFF)
1127 {
1128 iBit = (uMsr - 0xC0000000);
1129 pbMsrBitmap += 0x400;
1130 }
1131 else
1132 {
1133 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1134 return;
1135 }
1136
1137 Assert(iBit <= 0x1fff);
1138 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1139 ASMBitSet(pbMsrBitmap, iBit);
1140 else
1141 ASMBitClear(pbMsrBitmap, iBit);
1142
1143 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1144 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1145 else
1146 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1147}
1148
1149
1150#ifdef VBOX_STRICT
1151/**
1152 * Gets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @returns VBox status code.
1155 * @retval VINF_SUCCESS if the specified MSR is found.
1156 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1157 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1158 *
1159 * @param pVCpu Pointer to the VMCPU.
1160 * @param uMsr The MSR.
1161 * @param penmRead Where to store the read permissions.
1162 * @param penmWrite Where to store the write permissions.
1163 */
1164static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1165{
1166 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1167 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1168 int32_t iBit;
1169 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1170
1171 /* See hmR0VmxSetMsrPermission() for the layout. */
1172 if (uMsr <= 0x00001FFF)
1173 iBit = uMsr;
1174 else if ( uMsr >= 0xC0000000
1175 && uMsr <= 0xC0001FFF)
1176 {
1177 iBit = (uMsr - 0xC0000000);
1178 pbMsrBitmap += 0x400;
1179 }
1180 else
1181 {
1182 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1183 return VERR_NOT_SUPPORTED;
1184 }
1185
1186 Assert(iBit <= 0x1fff);
1187 if (ASMBitTest(pbMsrBitmap, iBit))
1188 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1189 else
1190 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1191
1192 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1193 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1194 else
1195 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1196 return VINF_SUCCESS;
1197}
1198#endif /* VBOX_STRICT */
1199
1200
1201/**
1202 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1203 * area.
1204 *
1205 * @returns VBox status code.
1206 * @param pVCpu Pointer to the VMCPU.
1207 * @param cMsrs The number of MSRs.
1208 */
1209DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1210{
1211 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1212 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1213 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1214 {
1215 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1216 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1218 }
1219
1220 /* Update number of guest MSRs to load/store across the world-switch. */
1221 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223
1224 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1225 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1226
1227 /* Update the VCPU's copy of the MSR count. */
1228 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1229
1230 return VINF_SUCCESS;
1231}
1232
1233
1234/**
1235 * Adds a new (or updates the value of an existing) guest/host MSR
1236 * pair to be swapped during the world-switch as part of the
1237 * auto-load/store MSR area in the VMCS.
1238 *
1239 * @returns true if the MSR was added -and- its value was updated, false
1240 * otherwise.
1241 * @param pVCpu Pointer to the VMCPU.
1242 * @param uMsr The MSR.
1243 * @param uGuestMsr Value of the guest MSR.
1244 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1245 * necessary.
1246 */
1247static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1248{
1249 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1250 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1251 uint32_t i;
1252 for (i = 0; i < cMsrs; i++)
1253 {
1254 if (pGuestMsr->u32Msr == uMsr)
1255 break;
1256 pGuestMsr++;
1257 }
1258
1259 bool fAdded = false;
1260 if (i == cMsrs)
1261 {
1262 ++cMsrs;
1263 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1264 AssertRC(rc);
1265
1266 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1267 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1268 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1269
1270 fAdded = true;
1271 }
1272
1273 /* Update the MSR values in the auto-load/store MSR area. */
1274 pGuestMsr->u32Msr = uMsr;
1275 pGuestMsr->u64Value = uGuestMsrValue;
1276
1277 /* Create/update the MSR slot in the host MSR area. */
1278 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1279 pHostMsr += i;
1280 pHostMsr->u32Msr = uMsr;
1281
1282 /*
1283 * Update the host MSR only when requested by the caller AND when we're
1284 * adding it to the auto-load/store area. Otherwise, it would have been
1285 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1286 */
1287 bool fUpdatedMsrValue = false;
1288 if ( fAdded
1289 && fUpdateHostMsr)
1290 {
1291 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1292 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1293 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1294 fUpdatedMsrValue = true;
1295 }
1296
1297 return fUpdatedMsrValue;
1298}
1299
1300
1301/**
1302 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1303 * auto-load/store MSR area in the VMCS.
1304 *
1305 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1306 * area.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1316 for (uint32_t i = 0; i < cMsrs; i++)
1317 {
1318 /* Find the MSR. */
1319 if (pGuestMsr->u32Msr == uMsr)
1320 {
1321 /* If it's the last MSR, simply reduce the count. */
1322 if (i == cMsrs - 1)
1323 {
1324 --cMsrs;
1325 break;
1326 }
1327
1328 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1329 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1330 pLastGuestMsr += cMsrs;
1331 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1332 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1333
1334 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1336 pLastHostMsr += cMsrs;
1337 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1338 pHostMsr->u64Value = pLastHostMsr->u64Value;
1339 --cMsrs;
1340 break;
1341 }
1342 pGuestMsr++;
1343 }
1344
1345 /* Update the VMCS if the count changed (meaning the MSR was found). */
1346 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1347 {
1348 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1349 AssertRCReturn(rc, rc);
1350
1351 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1352 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1353 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1354 }
1355
1356 return VINF_SUCCESS;
1357}
1358
1359
1360/**
1361 * Checks if the specified guest MSR is part of the auto-load/store area in
1362 * the VMCS.
1363 *
1364 * @returns true if found, false otherwise.
1365 * @param pVCpu Pointer to the VMCPU.
1366 * @param uMsr The MSR to find.
1367 */
1368static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1369{
1370 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1371 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1372
1373 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1374 {
1375 if (pGuestMsr->u32Msr == uMsr)
1376 return true;
1377 }
1378 return false;
1379}
1380
1381
1382/**
1383 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1384 *
1385 * @param pVCpu Pointer to the VMCPU.
1386 *
1387 * @remarks No-long-jump zone!!!
1388 */
1389static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1390{
1391 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1392 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1393 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1394 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1395
1396 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1397 {
1398 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1399 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1400 }
1401
1402 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1403}
1404
1405
1406#if HC_ARCH_BITS == 64
1407/**
1408 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1409 * perform lazy restoration of the host MSRs while leaving VT-x.
1410 *
1411 * @param pVCpu Pointer to the VMCPU.
1412 *
1413 * @remarks No-long-jump zone!!!
1414 */
1415static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1416{
1417 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1418
1419 /*
1420 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1421 */
1422 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1423 {
1424 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1425 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1426 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1427 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1428 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1429 }
1430}
1431
1432
1433/**
1434 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1435 * lazily while leaving VT-x.
1436 *
1437 * @returns true if it does, false otherwise.
1438 * @param pVCpu Pointer to the VMCPU.
1439 * @param uMsr The MSR to check.
1440 */
1441static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1442{
1443 NOREF(pVCpu);
1444 switch (uMsr)
1445 {
1446 case MSR_K8_LSTAR:
1447 case MSR_K6_STAR:
1448 case MSR_K8_SF_MASK:
1449 case MSR_K8_KERNEL_GS_BASE:
1450 return true;
1451 }
1452 return false;
1453}
1454
1455
1456/**
1457 * Saves a set of guests MSRs back into the guest-CPU context.
1458 *
1459 * @param pVCpu Pointer to the VMCPU.
1460 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1461 * out-of-sync. Make sure to update the required fields
1462 * before using them.
1463 *
1464 * @remarks No-long-jump zone!!!
1465 */
1466static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1467{
1468 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1469 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1470
1471 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1472 {
1473 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1474 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1475 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1476 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1477 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1478 }
1479}
1480
1481
1482/**
1483 * Loads a set of guests MSRs to allow read/passthru to the guest.
1484 *
1485 * The name of this function is slightly confusing. This function does NOT
1486 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1487 * common prefix for functions dealing with "lazy restoration" of the shared
1488 * MSRs.
1489 *
1490 * @param pVCpu Pointer to the VMCPU.
1491 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1492 * out-of-sync. Make sure to update the required fields
1493 * before using them.
1494 *
1495 * @remarks No-long-jump zone!!!
1496 */
1497static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1498{
1499 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1500 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1501
1502 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1503 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1504 {
1505#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1506 do { \
1507 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1508 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1509 else \
1510 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1511 } while (0)
1512
1513 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1514 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1515 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1516 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1517#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1518 }
1519 else
1520 {
1521 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1522 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1523 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1524 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1525 }
1526 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1527}
1528
1529
1530/**
1531 * Performs lazy restoration of the set of host MSRs if they were previously
1532 * loaded with guest MSR values.
1533 *
1534 * @param pVCpu Pointer to the VMCPU.
1535 *
1536 * @remarks No-long-jump zone!!!
1537 * @remarks The guest MSRs should have been saved back into the guest-CPU
1538 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1539 */
1540static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1541{
1542 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1543 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1544
1545 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1546 {
1547 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1548 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1549 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1550 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1551 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1552 }
1553 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1554}
1555#endif /* HC_ARCH_BITS == 64 */
1556
1557
1558#ifdef VBOX_STRICT
1559/**
1560 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1561 * VMCS are correct.
1562 *
1563 * @param pVCpu Pointer to the VMCPU.
1564 */
1565static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1566{
1567 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1568
1569 /* Verify MSR counts in the VMCS are what we think it should be. */
1570 uint32_t cMsrs;
1571 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1572 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1573
1574 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1575 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1576
1577 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1578 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1579
1580 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1581 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1582 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1583 {
1584 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1585 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1586 pGuestMsr->u32Msr));
1587
1588 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1589 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1590 pHostMsr->u64Value, u64Msr));
1591
1592 /* Verify that the permissions are as expected in the MSR bitmap. */
1593 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1594 {
1595 VMXMSREXITREAD enmRead;
1596 VMXMSREXITWRITE enmWrite;
1597 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1598 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1599 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1600 pGuestMsr->u32Msr));
1601 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1602 pGuestMsr->u32Msr));
1603 }
1604 }
1605}
1606# endif /* VBOX_STRICT */
1607
1608
1609/**
1610 * Flushes the TLB using EPT.
1611 *
1612 * @returns VBox status code.
1613 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1614 * enmFlush).
1615 * @param enmFlush Type of flush.
1616 *
1617 * @remarks Caller is responsible for making sure this function is called only
1618 * when NestedPaging is supported and providing @a enmFlush that is
1619 * supported by the CPU.
1620 * @remarks Can be called with interrupts disabled.
1621 */
1622static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1623{
1624 uint64_t au64Descriptor[2];
1625 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1626 au64Descriptor[0] = 0;
1627 else
1628 {
1629 Assert(pVCpu);
1630 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1631 }
1632 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1633
1634 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1635 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1636 rc));
1637 if ( RT_SUCCESS(rc)
1638 && pVCpu)
1639 {
1640 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1641 }
1642}
1643
1644
1645/**
1646 * Flushes the TLB using VPID.
1647 *
1648 * @returns VBox status code.
1649 * @param pVM Pointer to the VM.
1650 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1651 * enmFlush).
1652 * @param enmFlush Type of flush.
1653 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1654 * on @a enmFlush).
1655 *
1656 * @remarks Can be called with interrupts disabled.
1657 */
1658static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1659{
1660 NOREF(pVM);
1661 AssertPtr(pVM);
1662 Assert(pVM->hm.s.vmx.fVpid);
1663
1664 uint64_t au64Descriptor[2];
1665 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1666 {
1667 au64Descriptor[0] = 0;
1668 au64Descriptor[1] = 0;
1669 }
1670 else
1671 {
1672 AssertPtr(pVCpu);
1673 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1674 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1675 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1676 au64Descriptor[1] = GCPtr;
1677 }
1678
1679 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1680 AssertMsg(rc == VINF_SUCCESS,
1681 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1682 if ( RT_SUCCESS(rc)
1683 && pVCpu)
1684 {
1685 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1686 }
1687}
1688
1689
1690/**
1691 * Invalidates a guest page by guest virtual address. Only relevant for
1692 * EPT/VPID, otherwise there is nothing really to invalidate.
1693 *
1694 * @returns VBox status code.
1695 * @param pVM Pointer to the VM.
1696 * @param pVCpu Pointer to the VMCPU.
1697 * @param GCVirt Guest virtual address of the page to invalidate.
1698 */
1699VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1700{
1701 AssertPtr(pVM);
1702 AssertPtr(pVCpu);
1703 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1704
1705 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1706 if (!fFlushPending)
1707 {
1708 /*
1709 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1710 * See @bugref{6043} and @bugref{6177}.
1711 *
1712 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1713 * function maybe called in a loop with individual addresses.
1714 */
1715 if (pVM->hm.s.vmx.fVpid)
1716 {
1717 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1718 {
1719 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1720 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1721 }
1722 else
1723 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1724 }
1725 else if (pVM->hm.s.fNestedPaging)
1726 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1727 }
1728
1729 return VINF_SUCCESS;
1730}
1731
1732
1733/**
1734 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1735 * otherwise there is nothing really to invalidate.
1736 *
1737 * @returns VBox status code.
1738 * @param pVM Pointer to the VM.
1739 * @param pVCpu Pointer to the VMCPU.
1740 * @param GCPhys Guest physical address of the page to invalidate.
1741 */
1742VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1743{
1744 NOREF(pVM); NOREF(GCPhys);
1745 LogFlowFunc(("%RGp\n", GCPhys));
1746
1747 /*
1748 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1749 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1750 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1751 */
1752 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1753 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1754 return VINF_SUCCESS;
1755}
1756
1757
1758/**
1759 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1760 * case where neither EPT nor VPID is supported by the CPU.
1761 *
1762 * @param pVM Pointer to the VM.
1763 * @param pVCpu Pointer to the VMCPU.
1764 * @param pCpu Pointer to the global HM struct.
1765 *
1766 * @remarks Called with interrupts disabled.
1767 */
1768static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1769{
1770 AssertPtr(pVCpu);
1771 AssertPtr(pCpu);
1772 NOREF(pVM);
1773
1774 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1775
1776 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1777#if 0
1778 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1779 pVCpu->hm.s.TlbShootdown.cPages = 0;
1780#endif
1781
1782 Assert(pCpu->idCpu != NIL_RTCPUID);
1783 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1784 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1785 pVCpu->hm.s.fForceTLBFlush = false;
1786 return;
1787}
1788
1789
1790/**
1791 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1792 *
1793 * @param pVM Pointer to the VM.
1794 * @param pVCpu Pointer to the VMCPU.
1795 * @param pCpu Pointer to the global HM CPU struct.
1796 * @remarks All references to "ASID" in this function pertains to "VPID" in
1797 * Intel's nomenclature. The reason is, to avoid confusion in compare
1798 * statements since the host-CPU copies are named "ASID".
1799 *
1800 * @remarks Called with interrupts disabled.
1801 */
1802static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1803{
1804#ifdef VBOX_WITH_STATISTICS
1805 bool fTlbFlushed = false;
1806# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1807# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1808 if (!fTlbFlushed) \
1809 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1810 } while (0)
1811#else
1812# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1813# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1814#endif
1815
1816 AssertPtr(pVM);
1817 AssertPtr(pCpu);
1818 AssertPtr(pVCpu);
1819 Assert(pCpu->idCpu != NIL_RTCPUID);
1820 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1821 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1822 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1823
1824
1825 /*
1826 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1827 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1828 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1829 */
1830 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1831 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1832 {
1833 ++pCpu->uCurrentAsid;
1834 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1835 {
1836 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1837 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1838 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1839 }
1840
1841 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1842 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1843 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1844
1845 /*
1846 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1847 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1848 */
1849 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1850 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1851 HMVMX_SET_TAGGED_TLB_FLUSHED();
1852 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1853 }
1854
1855 /* Check for explicit TLB shootdowns. */
1856 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1857 {
1858 /*
1859 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1860 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1861 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1862 * but not guest-physical mappings.
1863 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1864 */
1865 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1866 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1867 HMVMX_SET_TAGGED_TLB_FLUSHED();
1868 }
1869
1870 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1871 * where it is commented out. Support individual entry flushing
1872 * someday. */
1873#if 0
1874 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1875 {
1876 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1877
1878 /*
1879 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1880 * as supported by the CPU.
1881 */
1882 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1883 {
1884 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1885 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1886 }
1887 else
1888 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1889
1890 HMVMX_SET_TAGGED_TLB_FLUSHED();
1891 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1892 pVCpu->hm.s.TlbShootdown.cPages = 0;
1893 }
1894#endif
1895
1896 pVCpu->hm.s.fForceTLBFlush = false;
1897
1898 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1899
1900 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1901 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1902 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1903 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1904 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1905 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1906 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1907 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1908 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1909
1910 /* Update VMCS with the VPID. */
1911 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1912 AssertRC(rc);
1913
1914#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1915}
1916
1917
1918/**
1919 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1920 *
1921 * @returns VBox status code.
1922 * @param pVM Pointer to the VM.
1923 * @param pVCpu Pointer to the VMCPU.
1924 * @param pCpu Pointer to the global HM CPU struct.
1925 *
1926 * @remarks Called with interrupts disabled.
1927 */
1928static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1929{
1930 AssertPtr(pVM);
1931 AssertPtr(pVCpu);
1932 AssertPtr(pCpu);
1933 Assert(pCpu->idCpu != NIL_RTCPUID);
1934 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1935 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1936
1937 /*
1938 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1939 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1940 */
1941 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1942 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1943 {
1944 pVCpu->hm.s.fForceTLBFlush = true;
1945 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946 }
1947
1948 /* Check for explicit TLB shootdown flushes. */
1949 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1950 {
1951 pVCpu->hm.s.fForceTLBFlush = true;
1952 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1953 }
1954
1955 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958 if (pVCpu->hm.s.fForceTLBFlush)
1959 {
1960 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961 pVCpu->hm.s.fForceTLBFlush = false;
1962 }
1963 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1964 * where it is commented out. Support individual entry flushing
1965 * someday. */
1966#if 0
1967 else
1968 {
1969 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1970 {
1971 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1973 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1974 }
1975 else
1976 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1977
1978 pVCpu->hm.s.TlbShootdown.cPages = 0;
1979 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980 }
1981#endif
1982}
1983
1984
1985/**
1986 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1987 *
1988 * @returns VBox status code.
1989 * @param pVM Pointer to the VM.
1990 * @param pVCpu Pointer to the VMCPU.
1991 * @param pCpu Pointer to the global HM CPU struct.
1992 *
1993 * @remarks Called with interrupts disabled.
1994 */
1995static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1996{
1997 AssertPtr(pVM);
1998 AssertPtr(pVCpu);
1999 AssertPtr(pCpu);
2000 Assert(pCpu->idCpu != NIL_RTCPUID);
2001 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2002 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2003
2004 /*
2005 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2006 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2007 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2008 */
2009 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2010 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2011 {
2012 pVCpu->hm.s.fForceTLBFlush = true;
2013 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2014 }
2015
2016 /* Check for explicit TLB shootdown flushes. */
2017 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2018 {
2019 /*
2020 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2021 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2022 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2023 */
2024 pVCpu->hm.s.fForceTLBFlush = true;
2025 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2026 }
2027
2028 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2029 if (pVCpu->hm.s.fForceTLBFlush)
2030 {
2031 ++pCpu->uCurrentAsid;
2032 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2033 {
2034 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2035 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2036 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2037 }
2038
2039 pVCpu->hm.s.fForceTLBFlush = false;
2040 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2041 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2042 if (pCpu->fFlushAsidBeforeUse)
2043 {
2044 if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_SINGLE_CONTEXT)
2045 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2046 else if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_ALL_CONTEXTS)
2047 {
2048 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
2049 pCpu->fFlushAsidBeforeUse = false;
2050 }
2051 else
2052 {
2053 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2054 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2055 }
2056 }
2057 }
2058 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2059 * where it is commented out. Support individual entry flushing
2060 * someday. */
2061#if 0
2062 else
2063 {
2064 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2065 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2066 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2067 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2068
2069 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2070 {
2071 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2072 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2073 {
2074 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2075 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2076 }
2077 else
2078 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2079
2080 pVCpu->hm.s.TlbShootdown.cPages = 0;
2081 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2082 }
2083 else
2084 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2085 }
2086#endif
2087
2088 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2089 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2090 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2091 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2092 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2093 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2094 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2095
2096 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2097 AssertRC(rc);
2098}
2099
2100
2101/**
2102 * Flushes the guest TLB entry based on CPU capabilities.
2103 *
2104 * @param pVCpu Pointer to the VMCPU.
2105 * @param pCpu Pointer to the global HM CPU struct.
2106 */
2107DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2108{
2109#ifdef HMVMX_ALWAYS_FLUSH_TLB
2110 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2111#endif
2112 PVM pVM = pVCpu->CTX_SUFF(pVM);
2113 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2114 {
2115 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2116 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2117 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2118 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2119 default:
2120 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2121 break;
2122 }
2123
2124 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2125 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2126
2127 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2128}
2129
2130
2131/**
2132 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2133 * TLB entries from the host TLB before VM-entry.
2134 *
2135 * @returns VBox status code.
2136 * @param pVM Pointer to the VM.
2137 */
2138static int hmR0VmxSetupTaggedTlb(PVM pVM)
2139{
2140 /*
2141 * Determine optimal flush type for Nested Paging.
2142 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2143 * guest execution (see hmR3InitFinalizeR0()).
2144 */
2145 if (pVM->hm.s.fNestedPaging)
2146 {
2147 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2148 {
2149 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2150 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2151 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2152 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2153 else
2154 {
2155 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2156 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2157 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2158 }
2159
2160 /* Make sure the write-back cacheable memory type for EPT is supported. */
2161 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2162 {
2163 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2164 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2165 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2166 }
2167 }
2168 else
2169 {
2170 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2171 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2172 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2173 }
2174 }
2175
2176 /*
2177 * Determine optimal flush type for VPID.
2178 */
2179 if (pVM->hm.s.vmx.fVpid)
2180 {
2181 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2182 {
2183 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2184 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2185 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2186 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2187 else
2188 {
2189 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2190 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2191 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2192 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2193 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2194 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2195 pVM->hm.s.vmx.fVpid = false;
2196 }
2197 }
2198 else
2199 {
2200 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2201 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2202 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2203 pVM->hm.s.vmx.fVpid = false;
2204 }
2205 }
2206
2207 /*
2208 * Setup the handler for flushing tagged-TLBs.
2209 */
2210 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2211 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2212 else if (pVM->hm.s.fNestedPaging)
2213 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2214 else if (pVM->hm.s.vmx.fVpid)
2215 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2216 else
2217 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2218 return VINF_SUCCESS;
2219}
2220
2221
2222/**
2223 * Sets up pin-based VM-execution controls in the VMCS.
2224 *
2225 * @returns VBox status code.
2226 * @param pVM Pointer to the VM.
2227 * @param pVCpu Pointer to the VMCPU.
2228 */
2229static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2230{
2231 AssertPtr(pVM);
2232 AssertPtr(pVCpu);
2233
2234 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2235 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2236
2237 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2238 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2239 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2240
2241 /* Enable the VMX preemption timer. */
2242 if (pVM->hm.s.vmx.fUsePreemptTimer)
2243 {
2244 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2245 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2246 }
2247
2248 if ((val & zap) != val)
2249 {
2250 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2251 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2252 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255
2256 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2257 AssertRCReturn(rc, rc);
2258
2259 /* Update VCPU with the currently set pin-based VM-execution controls. */
2260 pVCpu->hm.s.vmx.u32PinCtls = val;
2261 return rc;
2262}
2263
2264
2265/**
2266 * Sets up processor-based VM-execution controls in the VMCS.
2267 *
2268 * @returns VBox status code.
2269 * @param pVM Pointer to the VM.
2270 * @param pVMCPU Pointer to the VMCPU.
2271 */
2272static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2273{
2274 AssertPtr(pVM);
2275 AssertPtr(pVCpu);
2276
2277 int rc = VERR_INTERNAL_ERROR_5;
2278 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2279 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2280
2281 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2282 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2283 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2284 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2285 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2286 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2287 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2288
2289 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2290 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2291 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2292 {
2293 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2294 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2295 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2296 }
2297
2298 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2299 if (!pVM->hm.s.fNestedPaging)
2300 {
2301 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2302 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2303 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2304 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2305 }
2306
2307 /* Use TPR shadowing if supported by the CPU. */
2308 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2309 {
2310 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2311 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2312 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2313 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2314 AssertRCReturn(rc, rc);
2315
2316 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2317 /* CR8 writes causes a VM-exit based on TPR threshold. */
2318 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2319 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2320 }
2321 else
2322 {
2323 /*
2324 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2325 * Set this control only for 64-bit guests.
2326 */
2327 if (pVM->hm.s.fAllow64BitGuests)
2328 {
2329 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2330 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2331 }
2332 }
2333
2334 /* Use MSR-bitmaps if supported by the CPU. */
2335 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2336 {
2337 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2338
2339 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2340 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2341 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2342 AssertRCReturn(rc, rc);
2343
2344 /*
2345 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2346 * automatically as dedicated fields in the VMCS.
2347 */
2348 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2349 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2350 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2351 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2352 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2353
2354#if HC_ARCH_BITS == 64
2355 /*
2356 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2357 */
2358 if (pVM->hm.s.fAllow64BitGuests)
2359 {
2360 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2361 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2362 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2363 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2364 }
2365#endif
2366 }
2367
2368 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2369 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2370 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2371
2372 if ((val & zap) != val)
2373 {
2374 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2375 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2376 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2377 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2378 }
2379
2380 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2381 AssertRCReturn(rc, rc);
2382
2383 /* Update VCPU with the currently set processor-based VM-execution controls. */
2384 pVCpu->hm.s.vmx.u32ProcCtls = val;
2385
2386 /*
2387 * Secondary processor-based VM-execution controls.
2388 */
2389 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2390 {
2391 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2392 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2393
2394 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2395 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2396
2397 if (pVM->hm.s.fNestedPaging)
2398 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2399 else
2400 {
2401 /*
2402 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2403 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2404 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2405 */
2406 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2407 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2408 }
2409
2410 if (pVM->hm.s.vmx.fVpid)
2411 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2412
2413 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2414 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2415
2416 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2417 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2418 * done dynamically. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2420 {
2421 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2422 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2423 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2424 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2425 AssertRCReturn(rc, rc);
2426 }
2427
2428 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2429 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2430
2431 if ((val & zap) != val)
2432 {
2433 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2434 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2435 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2436 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2437 }
2438
2439 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2440 AssertRCReturn(rc, rc);
2441
2442 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2443 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2444 }
2445 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2446 {
2447 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2448 "available\n"));
2449 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2450 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2451 }
2452
2453 return VINF_SUCCESS;
2454}
2455
2456
2457/**
2458 * Sets up miscellaneous (everything other than Pin & Processor-based
2459 * VM-execution) control fields in the VMCS.
2460 *
2461 * @returns VBox status code.
2462 * @param pVM Pointer to the VM.
2463 * @param pVCpu Pointer to the VMCPU.
2464 */
2465static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2466{
2467 NOREF(pVM);
2468 AssertPtr(pVM);
2469 AssertPtr(pVCpu);
2470
2471 int rc = VERR_GENERAL_FAILURE;
2472
2473 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2474#if 0
2475 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2477 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2478
2479 /*
2480 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2481 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2482 * We thus use the exception bitmap to control it rather than use both.
2483 */
2484 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2485 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2486
2487 /** @todo Explore possibility of using IO-bitmaps. */
2488 /* All IO & IOIO instructions cause VM-exits. */
2489 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2490 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2491
2492 /* Initialize the MSR-bitmap area. */
2493 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2494 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2495 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2496#endif
2497
2498 /* Setup MSR auto-load/store area. */
2499 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2500 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2501 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2502 AssertRCReturn(rc, rc);
2503 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2504 AssertRCReturn(rc, rc);
2505
2506 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2507 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2508 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2509 AssertRCReturn(rc, rc);
2510
2511 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2512 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2513 AssertRCReturn(rc, rc);
2514
2515 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2516#if 0
2517 /* Setup debug controls */
2518 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2519 AssertRCReturn(rc, rc);
2520 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2521 AssertRCReturn(rc, rc);
2522#endif
2523
2524 return rc;
2525}
2526
2527
2528/**
2529 * Sets up the initial exception bitmap in the VMCS based on static conditions
2530 * (i.e. conditions that cannot ever change after starting the VM).
2531 *
2532 * @returns VBox status code.
2533 * @param pVM Pointer to the VM.
2534 * @param pVCpu Pointer to the VMCPU.
2535 */
2536static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2537{
2538 AssertPtr(pVM);
2539 AssertPtr(pVCpu);
2540
2541 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2542
2543 uint32_t u32XcptBitmap = 0;
2544
2545 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2546 if (!pVM->hm.s.fNestedPaging)
2547 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2548
2549 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2550 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2551 AssertRCReturn(rc, rc);
2552 return rc;
2553}
2554
2555
2556/**
2557 * Sets up the initial guest-state mask. The guest-state mask is consulted
2558 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2559 * for the nested virtualization case (as it would cause a VM-exit).
2560 *
2561 * @param pVCpu Pointer to the VMCPU.
2562 */
2563static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2564{
2565 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2566 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2567 return VINF_SUCCESS;
2568}
2569
2570
2571/**
2572 * Does per-VM VT-x initialization.
2573 *
2574 * @returns VBox status code.
2575 * @param pVM Pointer to the VM.
2576 */
2577VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2578{
2579 LogFlowFunc(("pVM=%p\n", pVM));
2580
2581 int rc = hmR0VmxStructsAlloc(pVM);
2582 if (RT_FAILURE(rc))
2583 {
2584 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2585 return rc;
2586 }
2587
2588 return VINF_SUCCESS;
2589}
2590
2591
2592/**
2593 * Does per-VM VT-x termination.
2594 *
2595 * @returns VBox status code.
2596 * @param pVM Pointer to the VM.
2597 */
2598VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2599{
2600 LogFlowFunc(("pVM=%p\n", pVM));
2601
2602#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2603 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2604 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2605#endif
2606 hmR0VmxStructsFree(pVM);
2607 return VINF_SUCCESS;
2608}
2609
2610
2611/**
2612 * Sets up the VM for execution under VT-x.
2613 * This function is only called once per-VM during initialization.
2614 *
2615 * @returns VBox status code.
2616 * @param pVM Pointer to the VM.
2617 */
2618VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2619{
2620 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2621 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2622
2623 LogFlowFunc(("pVM=%p\n", pVM));
2624
2625 /*
2626 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2627 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2628 */
2629 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2630 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2631 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2632 || !pVM->hm.s.vmx.pRealModeTSS))
2633 {
2634 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2635 return VERR_INTERNAL_ERROR;
2636 }
2637
2638#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2639 /*
2640 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2641 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2642 */
2643 if ( pVM->hm.s.fAllow64BitGuests
2644 && !HMVMX_IS_64BIT_HOST_MODE())
2645 {
2646 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2647 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2648 }
2649#endif
2650
2651 /* Initialize these always, see hmR3InitFinalizeR0().*/
2652 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2653 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2654
2655 /* Setup the tagged-TLB flush handlers. */
2656 int rc = hmR0VmxSetupTaggedTlb(pVM);
2657 if (RT_FAILURE(rc))
2658 {
2659 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2660 return rc;
2661 }
2662
2663 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2664 {
2665 PVMCPU pVCpu = &pVM->aCpus[i];
2666 AssertPtr(pVCpu);
2667 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2668
2669 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2670 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2671
2672 /* Set revision dword at the beginning of the VMCS structure. */
2673 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2674
2675 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2676 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2677 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2678 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2679
2680 /* Load this VMCS as the current VMCS. */
2681 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2682 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2683 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2684
2685 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2686 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2687 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2688
2689 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2690 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2694 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2695 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2696
2697 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2698 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2699 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2700
2701 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2702 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2703 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2704
2705#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2706 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2707 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2708 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2709#endif
2710
2711 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2712 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2713 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2714 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2715
2716 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2717
2718 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2719 }
2720
2721 return VINF_SUCCESS;
2722}
2723
2724
2725/**
2726 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2727 * the VMCS.
2728 *
2729 * @returns VBox status code.
2730 * @param pVM Pointer to the VM.
2731 * @param pVCpu Pointer to the VMCPU.
2732 */
2733DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2734{
2735 NOREF(pVM); NOREF(pVCpu);
2736
2737 RTCCUINTREG uReg = ASMGetCR0();
2738 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2739 AssertRCReturn(rc, rc);
2740
2741#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2742 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2743 if (HMVMX_IS_64BIT_HOST_MODE())
2744 {
2745 uint64_t uRegCR3 = HMR0Get64bitCR3();
2746 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2747 }
2748 else
2749#endif
2750 {
2751 uReg = ASMGetCR3();
2752 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2753 }
2754 AssertRCReturn(rc, rc);
2755
2756 uReg = ASMGetCR4();
2757 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2758 AssertRCReturn(rc, rc);
2759 return rc;
2760}
2761
2762
2763#if HC_ARCH_BITS == 64
2764/**
2765 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2766 * requirements. See hmR0VmxSaveHostSegmentRegs().
2767 */
2768# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2769 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2770 { \
2771 bool fValidSelector = true; \
2772 if ((selValue) & X86_SEL_LDT) \
2773 { \
2774 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2775 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2776 } \
2777 if (fValidSelector) \
2778 { \
2779 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2780 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2781 } \
2782 (selValue) = 0; \
2783 }
2784#endif
2785
2786
2787/**
2788 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2789 * the host-state area in the VMCS.
2790 *
2791 * @returns VBox status code.
2792 * @param pVM Pointer to the VM.
2793 * @param pVCpu Pointer to the VMCPU.
2794 */
2795DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2796{
2797 NOREF(pVM);
2798 int rc = VERR_INTERNAL_ERROR_5;
2799
2800#if HC_ARCH_BITS == 64
2801 /*
2802 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2803 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2804 */
2805 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2806 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2807#endif
2808
2809 /*
2810 * Host DS, ES, FS and GS segment registers.
2811 */
2812#if HC_ARCH_BITS == 64
2813 RTSEL uSelDS = ASMGetDS();
2814 RTSEL uSelES = ASMGetES();
2815 RTSEL uSelFS = ASMGetFS();
2816 RTSEL uSelGS = ASMGetGS();
2817#else
2818 RTSEL uSelDS = 0;
2819 RTSEL uSelES = 0;
2820 RTSEL uSelFS = 0;
2821 RTSEL uSelGS = 0;
2822#endif
2823
2824 /* Recalculate which host-state bits need to be manually restored. */
2825 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2826
2827 /*
2828 * Host CS and SS segment registers.
2829 */
2830#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2831 RTSEL uSelCS;
2832 RTSEL uSelSS;
2833 if (HMVMX_IS_64BIT_HOST_MODE())
2834 {
2835 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2836 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2837 }
2838 else
2839 {
2840 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2841 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2842 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2843 }
2844#else
2845 RTSEL uSelCS = ASMGetCS();
2846 RTSEL uSelSS = ASMGetSS();
2847#endif
2848
2849 /*
2850 * Host TR segment register.
2851 */
2852 RTSEL uSelTR = ASMGetTR();
2853
2854#if HC_ARCH_BITS == 64
2855 /*
2856 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2857 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2858 */
2859 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2860 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2861 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2862 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2863# undef VMXLOCAL_ADJUST_HOST_SEG
2864#endif
2865
2866 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2867 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2868 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2869 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2870 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2871 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2872 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2873 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2874 Assert(uSelCS);
2875 Assert(uSelTR);
2876
2877 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2878#if 0
2879 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2880 Assert(uSelSS != 0);
2881#endif
2882
2883 /* Write these host selector fields into the host-state area in the VMCS. */
2884 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2885 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2886#if HC_ARCH_BITS == 64
2887 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2888 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2889 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2890 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2891#endif
2892 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2893
2894 /*
2895 * Host GDTR and IDTR.
2896 */
2897 RTGDTR Gdtr;
2898 RT_ZERO(Gdtr);
2899#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2900 if (HMVMX_IS_64BIT_HOST_MODE())
2901 {
2902 X86XDTR64 Gdtr64;
2903 X86XDTR64 Idtr64;
2904 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2905 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2906 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2907
2908 Gdtr.cbGdt = Gdtr64.cb;
2909 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2910 }
2911 else
2912#endif
2913 {
2914 RTIDTR Idtr;
2915 ASMGetGDTR(&Gdtr);
2916 ASMGetIDTR(&Idtr);
2917 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2918 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2919
2920#if HC_ARCH_BITS == 64
2921 /*
2922 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2923 * maximum limit (0xffff) on every VM-exit.
2924 */
2925 if (Gdtr.cbGdt != 0xffff)
2926 {
2927 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2928 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2929 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2930 }
2931
2932 /*
2933 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2934 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2935 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2936 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2937 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2938 * hosts where we are pretty sure it won't cause trouble.
2939 */
2940# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2941 if (Idtr.cbIdt < 0x0fff)
2942# else
2943 if (Idtr.cbIdt != 0xffff)
2944# endif
2945 {
2946 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2947 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2948 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2949 }
2950#endif
2951 }
2952
2953 /*
2954 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2955 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2956 */
2957 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2958 {
2959 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2960 return VERR_VMX_INVALID_HOST_STATE;
2961 }
2962
2963 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2964#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2965 if (HMVMX_IS_64BIT_HOST_MODE())
2966 {
2967 /* We need the 64-bit TR base for hybrid darwin. */
2968 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2969 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2970 }
2971 else
2972#endif
2973 {
2974 uintptr_t uTRBase;
2975#if HC_ARCH_BITS == 64
2976 uTRBase = X86DESC64_BASE(pDesc);
2977
2978 /*
2979 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2980 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2981 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2982 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2983 *
2984 * [1] See Intel spec. 3.5 "System Descriptor Types".
2985 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2986 */
2987 Assert(pDesc->System.u4Type == 11);
2988 if ( pDesc->System.u16LimitLow != 0x67
2989 || pDesc->System.u4LimitHigh)
2990 {
2991 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2992 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2993
2994 /* Store the GDTR here as we need it while restoring TR. */
2995 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2996 }
2997#else
2998 uTRBase = X86DESC_BASE(pDesc);
2999#endif
3000 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3001 }
3002 AssertRCReturn(rc, rc);
3003
3004 /*
3005 * Host FS base and GS base.
3006 */
3007#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3011 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3012 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3014
3015# if HC_ARCH_BITS == 64
3016 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3017 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3018 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3019 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3020 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3021# endif
3022 }
3023#endif
3024 return rc;
3025}
3026
3027
3028/**
3029 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3030 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3031 * the host after every successful VM-exit.
3032 *
3033 * @returns VBox status code.
3034 * @param pVM Pointer to the VM.
3035 * @param pVCpu Pointer to the VMCPU.
3036 *
3037 * @remarks No-long-jump zone!!!
3038 */
3039DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3040{
3041 NOREF(pVM);
3042
3043 AssertPtr(pVCpu);
3044 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3045
3046 int rc = VINF_SUCCESS;
3047#if HC_ARCH_BITS == 64
3048 if (pVM->hm.s.fAllow64BitGuests)
3049 hmR0VmxLazySaveHostMsrs(pVCpu);
3050#endif
3051
3052 /*
3053 * Host Sysenter MSRs.
3054 */
3055 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3056 AssertRCReturn(rc, rc);
3057#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3058 if (HMVMX_IS_64BIT_HOST_MODE())
3059 {
3060 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3061 AssertRCReturn(rc, rc);
3062 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3063 }
3064 else
3065 {
3066 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3067 AssertRCReturn(rc, rc);
3068 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3069 }
3070#elif HC_ARCH_BITS == 32
3071 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3072 AssertRCReturn(rc, rc);
3073 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3074#else
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3076 AssertRCReturn(rc, rc);
3077 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3078#endif
3079 AssertRCReturn(rc, rc);
3080
3081 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3082 * hmR0VmxSetupExitCtls() !! */
3083 return rc;
3084}
3085
3086
3087/**
3088 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3089 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3090 * controls".
3091 *
3092 * @returns VBox status code.
3093 * @param pVCpu Pointer to the VMCPU.
3094 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3095 * out-of-sync. Make sure to update the required fields
3096 * before using them.
3097 *
3098 * @remarks No-long-jump zone!!!
3099 */
3100DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3101{
3102 int rc = VINF_SUCCESS;
3103 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3104 {
3105 PVM pVM = pVCpu->CTX_SUFF(pVM);
3106 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3107 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3108
3109 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3110 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3111
3112 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3113 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3114 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3115 else
3116 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3117
3118 /*
3119 * The following should -not- be set (since we're not in SMM mode):
3120 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3121 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3122 */
3123
3124 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3125 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3126 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3127
3128 if ((val & zap) != val)
3129 {
3130 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3131 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3132 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3133 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3134 }
3135
3136 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3137 AssertRCReturn(rc, rc);
3138
3139 /* Update VCPU with the currently set VM-exit controls. */
3140 pVCpu->hm.s.vmx.u32EntryCtls = val;
3141 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3142 }
3143 return rc;
3144}
3145
3146
3147/**
3148 * Sets up the VM-exit controls in the VMCS.
3149 *
3150 * @returns VBox status code.
3151 * @param pVM Pointer to the VM.
3152 * @param pVCpu Pointer to the VMCPU.
3153 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3154 * out-of-sync. Make sure to update the required fields
3155 * before using them.
3156 *
3157 * @remarks requires EFER.
3158 */
3159DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3160{
3161 NOREF(pMixedCtx);
3162
3163 int rc = VINF_SUCCESS;
3164 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3165 {
3166 PVM pVM = pVCpu->CTX_SUFF(pVM);
3167 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3168 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3169
3170 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3171 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3172
3173 /*
3174 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3175 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3176 */
3177#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3178 if (HMVMX_IS_64BIT_HOST_MODE())
3179 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3180 else
3181 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3182#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3183 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3184 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3185 else
3186 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3187#endif
3188
3189 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3190 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3191
3192 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3193 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3194 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3195 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3196 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3197
3198 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3199 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3200
3201 if ((val & zap) != val)
3202 {
3203 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3204 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3205 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3206 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3207 }
3208
3209 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3210 AssertRCReturn(rc, rc);
3211
3212 /* Update VCPU with the currently set VM-exit controls. */
3213 pVCpu->hm.s.vmx.u32ExitCtls = val;
3214 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3215 }
3216 return rc;
3217}
3218
3219
3220/**
3221 * Loads the guest APIC and related state.
3222 *
3223 * @returns VBox status code.
3224 * @param pVM Pointer to the VM.
3225 * @param pVCpu Pointer to the VMCPU.
3226 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3227 * out-of-sync. Make sure to update the required fields
3228 * before using them.
3229 */
3230DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3231{
3232 NOREF(pMixedCtx);
3233
3234 int rc = VINF_SUCCESS;
3235 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3236 {
3237 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3238 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3239 {
3240 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3241
3242 bool fPendingIntr = false;
3243 uint8_t u8Tpr = 0;
3244 uint8_t u8PendingIntr = 0;
3245 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3246 AssertRCReturn(rc, rc);
3247
3248 /*
3249 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3250 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3251 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3252 * the interrupt when we VM-exit for other reasons.
3253 */
3254 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3255 uint32_t u32TprThreshold = 0;
3256 if (fPendingIntr)
3257 {
3258 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3259 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3260 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3261 if (u8PendingPriority <= u8TprPriority)
3262 u32TprThreshold = u8PendingPriority;
3263 else
3264 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3265 }
3266 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3267
3268 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3269 AssertRCReturn(rc, rc);
3270 }
3271
3272 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3273 }
3274 return rc;
3275}
3276
3277
3278/**
3279 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3280 *
3281 * @returns Guest's interruptibility-state.
3282 * @param pVCpu Pointer to the VMCPU.
3283 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3284 * out-of-sync. Make sure to update the required fields
3285 * before using them.
3286 *
3287 * @remarks No-long-jump zone!!!
3288 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3289 */
3290DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3291{
3292 /*
3293 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3294 * inhibit interrupts or clear any existing interrupt-inhibition.
3295 */
3296 uint32_t uIntrState = 0;
3297 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3298 {
3299 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3300 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3301 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3302 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3303 {
3304 /*
3305 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3306 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3307 */
3308 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3309 }
3310 else if (pMixedCtx->eflags.Bits.u1IF)
3311 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3312 else
3313 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3314 }
3315 return uIntrState;
3316}
3317
3318
3319/**
3320 * Loads the guest's interruptibility-state into the guest-state area in the
3321 * VMCS.
3322 *
3323 * @returns VBox status code.
3324 * @param pVCpu Pointer to the VMCPU.
3325 * @param uIntrState The interruptibility-state to set.
3326 */
3327static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3328{
3329 NOREF(pVCpu);
3330 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3331 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3332 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3333 AssertRCReturn(rc, rc);
3334 return rc;
3335}
3336
3337
3338/**
3339 * Loads the guest's RIP into the guest-state area in the VMCS.
3340 *
3341 * @returns VBox status code.
3342 * @param pVCpu Pointer to the VMCPU.
3343 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3344 * out-of-sync. Make sure to update the required fields
3345 * before using them.
3346 *
3347 * @remarks No-long-jump zone!!!
3348 */
3349static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3350{
3351 int rc = VINF_SUCCESS;
3352 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3353 {
3354 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3355 AssertRCReturn(rc, rc);
3356
3357 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3358 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3359 }
3360 return rc;
3361}
3362
3363
3364/**
3365 * Loads the guest's RSP into the guest-state area in the VMCS.
3366 *
3367 * @returns VBox status code.
3368 * @param pVCpu Pointer to the VMCPU.
3369 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3370 * out-of-sync. Make sure to update the required fields
3371 * before using them.
3372 *
3373 * @remarks No-long-jump zone!!!
3374 */
3375static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3376{
3377 int rc = VINF_SUCCESS;
3378 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3379 {
3380 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3381 AssertRCReturn(rc, rc);
3382
3383 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3384 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3385 }
3386 return rc;
3387}
3388
3389
3390/**
3391 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3392 *
3393 * @returns VBox status code.
3394 * @param pVCpu Pointer to the VMCPU.
3395 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3396 * out-of-sync. Make sure to update the required fields
3397 * before using them.
3398 *
3399 * @remarks No-long-jump zone!!!
3400 */
3401static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3402{
3403 int rc = VINF_SUCCESS;
3404 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3405 {
3406 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3407 Let us assert it as such and use 32-bit VMWRITE. */
3408 Assert(!(pMixedCtx->rflags.u64 >> 32));
3409 X86EFLAGS Eflags = pMixedCtx->eflags;
3410 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3411 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3412
3413 /*
3414 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3415 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3416 */
3417 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3418 {
3419 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3420 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3421 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3422 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3423 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3424 }
3425
3426 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3427 AssertRCReturn(rc, rc);
3428
3429 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3430 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3431 }
3432 return rc;
3433}
3434
3435
3436/**
3437 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3438 *
3439 * @returns VBox status code.
3440 * @param pVCpu Pointer to the VMCPU.
3441 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3442 * out-of-sync. Make sure to update the required fields
3443 * before using them.
3444 *
3445 * @remarks No-long-jump zone!!!
3446 */
3447DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3448{
3449 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3450 AssertRCReturn(rc, rc);
3451 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3452 AssertRCReturn(rc, rc);
3453 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3454 AssertRCReturn(rc, rc);
3455 return rc;
3456}
3457
3458
3459/**
3460 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3461 * CR0 is partially shared with the host and we have to consider the FPU bits.
3462 *
3463 * @returns VBox status code.
3464 * @param pVM Pointer to the VM.
3465 * @param pVCpu Pointer to the VMCPU.
3466 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3467 * out-of-sync. Make sure to update the required fields
3468 * before using them.
3469 *
3470 * @remarks No-long-jump zone!!!
3471 */
3472static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3473{
3474 /*
3475 * Guest CR0.
3476 * Guest FPU.
3477 */
3478 int rc = VINF_SUCCESS;
3479 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3480 {
3481 Assert(!(pMixedCtx->cr0 >> 32));
3482 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3483 PVM pVM = pVCpu->CTX_SUFF(pVM);
3484
3485 /* The guest's view (read access) of its CR0 is unblemished. */
3486 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3487 AssertRCReturn(rc, rc);
3488 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3489
3490 /* Setup VT-x's view of the guest CR0. */
3491 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3492 if (pVM->hm.s.fNestedPaging)
3493 {
3494 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3495 {
3496 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3497 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3498 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3499 }
3500 else
3501 {
3502 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3503 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3504 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3505 }
3506
3507 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3508 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3509 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3510
3511 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3512 AssertRCReturn(rc, rc);
3513 }
3514 else
3515 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3516
3517 /*
3518 * Guest FPU bits.
3519 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3520 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3521 */
3522 u32GuestCR0 |= X86_CR0_NE;
3523 bool fInterceptNM = false;
3524 if (CPUMIsGuestFPUStateActive(pVCpu))
3525 {
3526 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3527 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3528 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3529 }
3530 else
3531 {
3532 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3533 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3534 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3535 }
3536
3537 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3538 bool fInterceptMF = false;
3539 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3540 fInterceptMF = true;
3541
3542 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3543 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3544 {
3545 Assert(PDMVmmDevHeapIsEnabled(pVM));
3546 Assert(pVM->hm.s.vmx.pRealModeTSS);
3547 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3548 fInterceptNM = true;
3549 fInterceptMF = true;
3550 }
3551 else
3552 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3553
3554 if (fInterceptNM)
3555 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3556 else
3557 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3558
3559 if (fInterceptMF)
3560 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3561 else
3562 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3563
3564 /* Additional intercepts for debugging, define these yourself explicitly. */
3565#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3566 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3567 | RT_BIT(X86_XCPT_BP)
3568 | RT_BIT(X86_XCPT_DB)
3569 | RT_BIT(X86_XCPT_DE)
3570 | RT_BIT(X86_XCPT_NM)
3571 | RT_BIT(X86_XCPT_UD)
3572 | RT_BIT(X86_XCPT_NP)
3573 | RT_BIT(X86_XCPT_SS)
3574 | RT_BIT(X86_XCPT_GP)
3575 | RT_BIT(X86_XCPT_PF)
3576 | RT_BIT(X86_XCPT_MF)
3577 ;
3578#elif defined(HMVMX_ALWAYS_TRAP_PF)
3579 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3580#endif
3581
3582 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3583
3584 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3585 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3586 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3587 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3588 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3589 else
3590 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3591
3592 u32GuestCR0 |= uSetCR0;
3593 u32GuestCR0 &= uZapCR0;
3594 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3595
3596 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3597 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3598 AssertRCReturn(rc, rc);
3599 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3600 AssertRCReturn(rc, rc);
3601 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3602
3603 /*
3604 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3605 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3606 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3607 */
3608 uint32_t u32CR0Mask = 0;
3609 u32CR0Mask = X86_CR0_PE
3610 | X86_CR0_NE
3611 | X86_CR0_WP
3612 | X86_CR0_PG
3613 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3614 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3615 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3616
3617 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3618 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3619 * and @bugref{6944}. */
3620#if 0
3621 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3622 u32CR0Mask &= ~X86_CR0_PE;
3623#endif
3624 if (pVM->hm.s.fNestedPaging)
3625 u32CR0Mask &= ~X86_CR0_WP;
3626
3627 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3628 if (fInterceptNM)
3629 {
3630 u32CR0Mask |= X86_CR0_TS
3631 | X86_CR0_MP;
3632 }
3633
3634 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3635 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3636 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3637 AssertRCReturn(rc, rc);
3638 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3639
3640 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3641 }
3642 return rc;
3643}
3644
3645
3646/**
3647 * Loads the guest control registers (CR3, CR4) into the guest-state area
3648 * in the VMCS.
3649 *
3650 * @returns VBox status code.
3651 * @param pVM Pointer to the VM.
3652 * @param pVCpu Pointer to the VMCPU.
3653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3654 * out-of-sync. Make sure to update the required fields
3655 * before using them.
3656 *
3657 * @remarks No-long-jump zone!!!
3658 */
3659static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3660{
3661 int rc = VINF_SUCCESS;
3662 PVM pVM = pVCpu->CTX_SUFF(pVM);
3663
3664 /*
3665 * Guest CR2.
3666 * It's always loaded in the assembler code. Nothing to do here.
3667 */
3668
3669 /*
3670 * Guest CR3.
3671 */
3672 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3673 {
3674 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3675 if (pVM->hm.s.fNestedPaging)
3676 {
3677 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3678
3679 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3680 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3681 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3682 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3683
3684 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3685 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3686 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3687
3688 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3689 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3690 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3691 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3692
3693 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3694 AssertRCReturn(rc, rc);
3695 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3696
3697 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3698 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3699 {
3700 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3701 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3702 {
3703 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3704 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3705 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3706 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3707 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3708 }
3709
3710 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3711 have Unrestricted Execution to handle the guest when it's not using paging. */
3712 GCPhysGuestCR3 = pMixedCtx->cr3;
3713 }
3714 else
3715 {
3716 /*
3717 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3718 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3719 * EPT takes care of translating it to host-physical addresses.
3720 */
3721 RTGCPHYS GCPhys;
3722 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3723 Assert(PDMVmmDevHeapIsEnabled(pVM));
3724
3725 /* We obtain it here every time as the guest could have relocated this PCI region. */
3726 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3727 AssertRCReturn(rc, rc);
3728
3729 GCPhysGuestCR3 = GCPhys;
3730 }
3731
3732 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3733 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3734 }
3735 else
3736 {
3737 /* Non-nested paging case, just use the hypervisor's CR3. */
3738 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3739
3740 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3741 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3742 }
3743 AssertRCReturn(rc, rc);
3744
3745 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3746 }
3747
3748 /*
3749 * Guest CR4.
3750 */
3751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3752 {
3753 Assert(!(pMixedCtx->cr4 >> 32));
3754 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3755
3756 /* The guest's view of its CR4 is unblemished. */
3757 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3758 AssertRCReturn(rc, rc);
3759 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3760
3761 /* Setup VT-x's view of the guest CR4. */
3762 /*
3763 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3764 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3765 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3766 */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(pVM->hm.s.vmx.pRealModeTSS);
3770 Assert(PDMVmmDevHeapIsEnabled(pVM));
3771 u32GuestCR4 &= ~X86_CR4_VME;
3772 }
3773
3774 if (pVM->hm.s.fNestedPaging)
3775 {
3776 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3777 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3778 {
3779 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3780 u32GuestCR4 |= X86_CR4_PSE;
3781 /* Our identity mapping is a 32-bit page directory. */
3782 u32GuestCR4 &= ~X86_CR4_PAE;
3783 }
3784 /* else use guest CR4.*/
3785 }
3786 else
3787 {
3788 /*
3789 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3790 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3791 */
3792 switch (pVCpu->hm.s.enmShadowMode)
3793 {
3794 case PGMMODE_REAL: /* Real-mode. */
3795 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3796 case PGMMODE_32_BIT: /* 32-bit paging. */
3797 {
3798 u32GuestCR4 &= ~X86_CR4_PAE;
3799 break;
3800 }
3801
3802 case PGMMODE_PAE: /* PAE paging. */
3803 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3804 {
3805 u32GuestCR4 |= X86_CR4_PAE;
3806 break;
3807 }
3808
3809 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3810 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3811#ifdef VBOX_ENABLE_64_BITS_GUESTS
3812 break;
3813#endif
3814 default:
3815 AssertFailed();
3816 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3817 }
3818 }
3819
3820 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3821 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3822 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3823 u32GuestCR4 |= uSetCR4;
3824 u32GuestCR4 &= uZapCR4;
3825
3826 /* Write VT-x's view of the guest CR4 into the VMCS. */
3827 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3828 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3829 AssertRCReturn(rc, rc);
3830
3831 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3832 uint32_t u32CR4Mask = 0;
3833 u32CR4Mask = X86_CR4_VME
3834 | X86_CR4_PAE
3835 | X86_CR4_PGE
3836 | X86_CR4_PSE
3837 | X86_CR4_VMXE;
3838 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3839 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3840 AssertRCReturn(rc, rc);
3841
3842 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3843 }
3844 return rc;
3845}
3846
3847
3848/**
3849 * Loads the guest debug registers into the guest-state area in the VMCS.
3850 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3851 *
3852 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3853 *
3854 * @returns VBox status code.
3855 * @param pVCpu Pointer to the VMCPU.
3856 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3857 * out-of-sync. Make sure to update the required fields
3858 * before using them.
3859 *
3860 * @remarks No-long-jump zone!!!
3861 */
3862static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3863{
3864 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3865 return VINF_SUCCESS;
3866
3867#ifdef VBOX_STRICT
3868 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3869 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3870 {
3871 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3872 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3873 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3874 }
3875#endif
3876
3877 int rc;
3878 PVM pVM = pVCpu->CTX_SUFF(pVM);
3879 bool fInterceptDB = false;
3880 bool fInterceptMovDRx = false;
3881 if ( pVCpu->hm.s.fSingleInstruction
3882 || DBGFIsStepping(pVCpu))
3883 {
3884 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3885 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3886 {
3887 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3888 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3889 AssertRCReturn(rc, rc);
3890 Assert(fInterceptDB == false);
3891 }
3892 else
3893 {
3894 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3895 pVCpu->hm.s.fClearTrapFlag = true;
3896 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3897 fInterceptDB = true;
3898 }
3899 }
3900
3901 if ( fInterceptDB
3902 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3903 {
3904 /*
3905 * Use the combined guest and host DRx values found in the hypervisor
3906 * register set because the debugger has breakpoints active or someone
3907 * is single stepping on the host side without a monitor trap flag.
3908 *
3909 * Note! DBGF expects a clean DR6 state before executing guest code.
3910 */
3911#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3912 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3913 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3914 {
3915 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3916 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3917 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3918 }
3919 else
3920#endif
3921 if (!CPUMIsHyperDebugStateActive(pVCpu))
3922 {
3923 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3924 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3925 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3926 }
3927
3928 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3929 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3930 AssertRCReturn(rc, rc);
3931
3932 pVCpu->hm.s.fUsingHyperDR7 = true;
3933 fInterceptDB = true;
3934 fInterceptMovDRx = true;
3935 }
3936 else
3937 {
3938 /*
3939 * If the guest has enabled debug registers, we need to load them prior to
3940 * executing guest code so they'll trigger at the right time.
3941 */
3942 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3943 {
3944#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3945 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3946 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3947 {
3948 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3949 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3950 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3951 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3952 }
3953 else
3954#endif
3955 if (!CPUMIsGuestDebugStateActive(pVCpu))
3956 {
3957 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3958 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3959 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3960 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3961 }
3962 Assert(!fInterceptDB);
3963 Assert(!fInterceptMovDRx);
3964 }
3965 /*
3966 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3967 * must intercept #DB in order to maintain a correct DR6 guest value.
3968 */
3969#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3970 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3971 && !CPUMIsGuestDebugStateActive(pVCpu))
3972#else
3973 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3974#endif
3975 {
3976 fInterceptMovDRx = true;
3977 fInterceptDB = true;
3978 }
3979
3980 /* Update guest DR7. */
3981 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3982 AssertRCReturn(rc, rc);
3983
3984 pVCpu->hm.s.fUsingHyperDR7 = false;
3985 }
3986
3987 /*
3988 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3989 */
3990 if (fInterceptDB)
3991 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3992 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3993 {
3994#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3995 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3996#endif
3997 }
3998 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3999 AssertRCReturn(rc, rc);
4000
4001 /*
4002 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4003 */
4004 if (fInterceptMovDRx)
4005 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4006 else
4007 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4008 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4009 AssertRCReturn(rc, rc);
4010
4011 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4012 return VINF_SUCCESS;
4013}
4014
4015
4016#ifdef VBOX_STRICT
4017/**
4018 * Strict function to validate segment registers.
4019 *
4020 * @remarks ASSUMES CR0 is up to date.
4021 */
4022static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4023{
4024 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4025 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4026 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4027 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4028 && ( !CPUMIsGuestInRealModeEx(pCtx)
4029 && !CPUMIsGuestInV86ModeEx(pCtx)))
4030 {
4031 /* Protected mode checks */
4032 /* CS */
4033 Assert(pCtx->cs.Attr.n.u1Present);
4034 Assert(!(pCtx->cs.Attr.u & 0xf00));
4035 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4036 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4037 || !(pCtx->cs.Attr.n.u1Granularity));
4038 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4039 || (pCtx->cs.Attr.n.u1Granularity));
4040 /* CS cannot be loaded with NULL in protected mode. */
4041 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4042 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4043 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4044 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4045 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4046 else
4047 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4048 /* SS */
4049 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4050 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4051 if ( !(pCtx->cr0 & X86_CR0_PE)
4052 || pCtx->cs.Attr.n.u4Type == 3)
4053 {
4054 Assert(!pCtx->ss.Attr.n.u2Dpl);
4055 }
4056 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4057 {
4058 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4059 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4060 Assert(pCtx->ss.Attr.n.u1Present);
4061 Assert(!(pCtx->ss.Attr.u & 0xf00));
4062 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4063 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4064 || !(pCtx->ss.Attr.n.u1Granularity));
4065 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4066 || (pCtx->ss.Attr.n.u1Granularity));
4067 }
4068 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4069 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4070 {
4071 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4072 Assert(pCtx->ds.Attr.n.u1Present);
4073 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4074 Assert(!(pCtx->ds.Attr.u & 0xf00));
4075 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4076 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4077 || !(pCtx->ds.Attr.n.u1Granularity));
4078 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4079 || (pCtx->ds.Attr.n.u1Granularity));
4080 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4081 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4082 }
4083 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4084 {
4085 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4086 Assert(pCtx->es.Attr.n.u1Present);
4087 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4088 Assert(!(pCtx->es.Attr.u & 0xf00));
4089 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4090 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4091 || !(pCtx->es.Attr.n.u1Granularity));
4092 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4093 || (pCtx->es.Attr.n.u1Granularity));
4094 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4095 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4096 }
4097 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4098 {
4099 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4100 Assert(pCtx->fs.Attr.n.u1Present);
4101 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4102 Assert(!(pCtx->fs.Attr.u & 0xf00));
4103 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4104 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4105 || !(pCtx->fs.Attr.n.u1Granularity));
4106 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4107 || (pCtx->fs.Attr.n.u1Granularity));
4108 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4109 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4110 }
4111 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4112 {
4113 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4114 Assert(pCtx->gs.Attr.n.u1Present);
4115 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4116 Assert(!(pCtx->gs.Attr.u & 0xf00));
4117 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4118 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4119 || !(pCtx->gs.Attr.n.u1Granularity));
4120 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4121 || (pCtx->gs.Attr.n.u1Granularity));
4122 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4123 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4124 }
4125 /* 64-bit capable CPUs. */
4126# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4127 if (HMVMX_IS_64BIT_HOST_MODE())
4128 {
4129 Assert(!(pCtx->cs.u64Base >> 32));
4130 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4131 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4132 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4133 }
4134# endif
4135 }
4136 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4137 || ( CPUMIsGuestInRealModeEx(pCtx)
4138 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4139 {
4140 /* Real and v86 mode checks. */
4141 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4142 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4143 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4144 {
4145 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4146 }
4147 else
4148 {
4149 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4150 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4151 }
4152
4153 /* CS */
4154 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4155 Assert(pCtx->cs.u32Limit == 0xffff);
4156 Assert(u32CSAttr == 0xf3);
4157 /* SS */
4158 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4159 Assert(pCtx->ss.u32Limit == 0xffff);
4160 Assert(u32SSAttr == 0xf3);
4161 /* DS */
4162 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4163 Assert(pCtx->ds.u32Limit == 0xffff);
4164 Assert(u32DSAttr == 0xf3);
4165 /* ES */
4166 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4167 Assert(pCtx->es.u32Limit == 0xffff);
4168 Assert(u32ESAttr == 0xf3);
4169 /* FS */
4170 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4171 Assert(pCtx->fs.u32Limit == 0xffff);
4172 Assert(u32FSAttr == 0xf3);
4173 /* GS */
4174 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4175 Assert(pCtx->gs.u32Limit == 0xffff);
4176 Assert(u32GSAttr == 0xf3);
4177 /* 64-bit capable CPUs. */
4178# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4179 if (HMVMX_IS_64BIT_HOST_MODE())
4180 {
4181 Assert(!(pCtx->cs.u64Base >> 32));
4182 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4183 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4184 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4185 }
4186# endif
4187 }
4188}
4189#endif /* VBOX_STRICT */
4190
4191
4192/**
4193 * Writes a guest segment register into the guest-state area in the VMCS.
4194 *
4195 * @returns VBox status code.
4196 * @param pVCpu Pointer to the VMCPU.
4197 * @param idxSel Index of the selector in the VMCS.
4198 * @param idxLimit Index of the segment limit in the VMCS.
4199 * @param idxBase Index of the segment base in the VMCS.
4200 * @param idxAccess Index of the access rights of the segment in the VMCS.
4201 * @param pSelReg Pointer to the segment selector.
4202 *
4203 * @remarks No-long-jump zone!!!
4204 */
4205static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4206 uint32_t idxAccess, PCPUMSELREG pSelReg)
4207{
4208 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4209 AssertRCReturn(rc, rc);
4210 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4211 AssertRCReturn(rc, rc);
4212 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4213 AssertRCReturn(rc, rc);
4214
4215 uint32_t u32Access = pSelReg->Attr.u;
4216 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4217 {
4218 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4219 u32Access = 0xf3;
4220 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4221 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4222 }
4223 else
4224 {
4225 /*
4226 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4227 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4228 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4229 * loaded in protected-mode have their attribute as 0.
4230 */
4231 if (!u32Access)
4232 u32Access = X86DESCATTR_UNUSABLE;
4233 }
4234
4235 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4236 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4237 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4238
4239 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4240 AssertRCReturn(rc, rc);
4241 return rc;
4242}
4243
4244
4245/**
4246 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4247 * into the guest-state area in the VMCS.
4248 *
4249 * @returns VBox status code.
4250 * @param pVM Pointer to the VM.
4251 * @param pVCPU Pointer to the VMCPU.
4252 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4253 * out-of-sync. Make sure to update the required fields
4254 * before using them.
4255 *
4256 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4257 * @remarks No-long-jump zone!!!
4258 */
4259static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4260{
4261 int rc = VERR_INTERNAL_ERROR_5;
4262 PVM pVM = pVCpu->CTX_SUFF(pVM);
4263
4264 /*
4265 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4266 */
4267 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4268 {
4269 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4270 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4271 {
4272 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4273 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4274 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4275 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4276 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4277 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4278 }
4279
4280#ifdef VBOX_WITH_REM
4281 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4282 {
4283 Assert(pVM->hm.s.vmx.pRealModeTSS);
4284 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4285 if ( pVCpu->hm.s.vmx.fWasInRealMode
4286 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4287 {
4288 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4289 in real-mode (e.g. OpenBSD 4.0) */
4290 REMFlushTBs(pVM);
4291 Log4(("Load: Switch to protected mode detected!\n"));
4292 pVCpu->hm.s.vmx.fWasInRealMode = false;
4293 }
4294 }
4295#endif
4296 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4297 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4298 AssertRCReturn(rc, rc);
4299 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4300 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4301 AssertRCReturn(rc, rc);
4302 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4303 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4304 AssertRCReturn(rc, rc);
4305 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4306 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4307 AssertRCReturn(rc, rc);
4308 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4309 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4310 AssertRCReturn(rc, rc);
4311 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4312 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4313 AssertRCReturn(rc, rc);
4314
4315#ifdef VBOX_STRICT
4316 /* Validate. */
4317 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4318#endif
4319
4320 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4321 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4322 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4323 }
4324
4325 /*
4326 * Guest TR.
4327 */
4328 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4329 {
4330 /*
4331 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4332 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4333 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4334 */
4335 uint16_t u16Sel = 0;
4336 uint32_t u32Limit = 0;
4337 uint64_t u64Base = 0;
4338 uint32_t u32AccessRights = 0;
4339
4340 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4341 {
4342 u16Sel = pMixedCtx->tr.Sel;
4343 u32Limit = pMixedCtx->tr.u32Limit;
4344 u64Base = pMixedCtx->tr.u64Base;
4345 u32AccessRights = pMixedCtx->tr.Attr.u;
4346 }
4347 else
4348 {
4349 Assert(pVM->hm.s.vmx.pRealModeTSS);
4350 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4351
4352 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4353 RTGCPHYS GCPhys;
4354 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4355 AssertRCReturn(rc, rc);
4356
4357 X86DESCATTR DescAttr;
4358 DescAttr.u = 0;
4359 DescAttr.n.u1Present = 1;
4360 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4361
4362 u16Sel = 0;
4363 u32Limit = HM_VTX_TSS_SIZE;
4364 u64Base = GCPhys; /* in real-mode phys = virt. */
4365 u32AccessRights = DescAttr.u;
4366 }
4367
4368 /* Validate. */
4369 Assert(!(u16Sel & RT_BIT(2)));
4370 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4371 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4372 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4373 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4374 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4375 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4376 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4377 Assert( (u32Limit & 0xfff) == 0xfff
4378 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4379 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4380 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4381
4382 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4383 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4384 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4385 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4386
4387 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4388 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4389 }
4390
4391 /*
4392 * Guest GDTR.
4393 */
4394 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4395 {
4396 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4397 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4398
4399 /* Validate. */
4400 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4401
4402 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4403 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4404 }
4405
4406 /*
4407 * Guest LDTR.
4408 */
4409 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4410 {
4411 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4412 uint32_t u32Access = 0;
4413 if (!pMixedCtx->ldtr.Attr.u)
4414 u32Access = X86DESCATTR_UNUSABLE;
4415 else
4416 u32Access = pMixedCtx->ldtr.Attr.u;
4417
4418 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4419 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4420 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4421 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4422
4423 /* Validate. */
4424 if (!(u32Access & X86DESCATTR_UNUSABLE))
4425 {
4426 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4427 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4428 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4429 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4430 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4431 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4432 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4433 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4434 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4435 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4436 }
4437
4438 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4439 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4440 }
4441
4442 /*
4443 * Guest IDTR.
4444 */
4445 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4446 {
4447 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4448 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4449
4450 /* Validate. */
4451 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4452
4453 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4454 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4455 }
4456
4457 return VINF_SUCCESS;
4458}
4459
4460
4461/**
4462 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4463 * areas. These MSRs will automatically be loaded to the host CPU on every
4464 * successful VM entry and stored from the host CPU on every successful VM-exit.
4465 *
4466 * This also creates/updates MSR slots for the host MSRs. The actual host
4467 * MSR values are -not- updated here for performance reasons. See
4468 * hmR0VmxSaveHostMsrs().
4469 *
4470 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4471 *
4472 * @returns VBox status code.
4473 * @param pVCpu Pointer to the VMCPU.
4474 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4475 * out-of-sync. Make sure to update the required fields
4476 * before using them.
4477 *
4478 * @remarks No-long-jump zone!!!
4479 */
4480static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4481{
4482 AssertPtr(pVCpu);
4483 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4484
4485 /*
4486 * MSRs that we use the auto-load/store MSR area in the VMCS.
4487 */
4488 PVM pVM = pVCpu->CTX_SUFF(pVM);
4489 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4490 {
4491#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4492 if (pVM->hm.s.fAllow64BitGuests)
4493 {
4494 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4495 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4496 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4497 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4498# ifdef DEBUG
4499 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4500 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4501 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4502# endif
4503 }
4504#endif
4505 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4506 }
4507
4508 /*
4509 * Guest Sysenter MSRs.
4510 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4511 * VM-exits on WRMSRs for these MSRs.
4512 */
4513 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4514 {
4515 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4516 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4517 }
4518
4519 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4520 {
4521 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4522 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4523 }
4524
4525 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4526 {
4527 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4528 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4529 }
4530
4531 return VINF_SUCCESS;
4532}
4533
4534
4535/**
4536 * Loads the guest activity state into the guest-state area in the VMCS.
4537 *
4538 * @returns VBox status code.
4539 * @param pVCpu Pointer to the VMCPU.
4540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4541 * out-of-sync. Make sure to update the required fields
4542 * before using them.
4543 *
4544 * @remarks No-long-jump zone!!!
4545 */
4546static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4547{
4548 NOREF(pCtx);
4549 /** @todo See if we can make use of other states, e.g.
4550 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4551 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4552 {
4553 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4554 AssertRCReturn(rc, rc);
4555
4556 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4557 }
4558 return VINF_SUCCESS;
4559}
4560
4561
4562/**
4563 * Sets up the appropriate function to run guest code.
4564 *
4565 * @returns VBox status code.
4566 * @param pVCpu Pointer to the VMCPU.
4567 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4568 * out-of-sync. Make sure to update the required fields
4569 * before using them.
4570 *
4571 * @remarks No-long-jump zone!!!
4572 */
4573static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4574{
4575 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4576 {
4577#ifndef VBOX_ENABLE_64_BITS_GUESTS
4578 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4579#endif
4580 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4581#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4582 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4583 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4584 {
4585 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4586 {
4587 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4588 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS),
4589 ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4590 }
4591 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4592 }
4593#else
4594 /* 64-bit host or hybrid host. */
4595 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4596#endif
4597 }
4598 else
4599 {
4600 /* Guest is not in long mode, use the 32-bit handler. */
4601#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4602 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4603 {
4604 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4605 {
4606 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4607 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS),
4608 ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4609 }
4610 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4611 }
4612#else
4613 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4614#endif
4615 }
4616 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4617 return VINF_SUCCESS;
4618}
4619
4620
4621/**
4622 * Wrapper for running the guest code in VT-x.
4623 *
4624 * @returns VBox strict status code.
4625 * @param pVM Pointer to the VM.
4626 * @param pVCpu Pointer to the VMCPU.
4627 * @param pCtx Pointer to the guest-CPU context.
4628 *
4629 * @remarks No-long-jump zone!!!
4630 */
4631DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4632{
4633 /*
4634 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4635 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4636 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4637 */
4638 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4639 /** @todo Add stats for resume vs launch. */
4640#ifdef VBOX_WITH_KERNEL_USING_XMM
4641 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4642#else
4643 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4644#endif
4645}
4646
4647
4648/**
4649 * Reports world-switch error and dumps some useful debug info.
4650 *
4651 * @param pVM Pointer to the VM.
4652 * @param pVCpu Pointer to the VMCPU.
4653 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4654 * @param pCtx Pointer to the guest-CPU context.
4655 * @param pVmxTransient Pointer to the VMX transient structure (only
4656 * exitReason updated).
4657 */
4658static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4659{
4660 Assert(pVM);
4661 Assert(pVCpu);
4662 Assert(pCtx);
4663 Assert(pVmxTransient);
4664 HMVMX_ASSERT_PREEMPT_SAFE();
4665
4666 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4667 switch (rcVMRun)
4668 {
4669 case VERR_VMX_INVALID_VMXON_PTR:
4670 AssertFailed();
4671 break;
4672 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4673 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4674 {
4675 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4676 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4677 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4678 AssertRC(rc);
4679
4680 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4681 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4682 Cannot do it here as we may have been long preempted. */
4683
4684#ifdef VBOX_STRICT
4685 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4686 pVmxTransient->uExitReason));
4687 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4688 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4689 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4690 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4691 else
4692 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4693 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4694 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4695
4696 /* VMX control bits. */
4697 uint32_t u32Val;
4698 uint64_t u64Val;
4699 HMVMXHCUINTREG uHCReg;
4700 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4701 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4702 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4703 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4704 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4705 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4707 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4708 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4709 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4710 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4711 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4712 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4713 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4715 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4716 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4717 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4718 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4719 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4721 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4722 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4723 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4724 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4725 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4726 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4727 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4728 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4729 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4730 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4731 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4732 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4733 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4734 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4735 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4736 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4737 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4738 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4739 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4740 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4741 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4742
4743 /* Guest bits. */
4744 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4745 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4746 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4747 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4748 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4749 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4750 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4751 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4752
4753 /* Host bits. */
4754 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4755 Log4(("Host CR0 %#RHr\n", uHCReg));
4756 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4757 Log4(("Host CR3 %#RHr\n", uHCReg));
4758 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4759 Log4(("Host CR4 %#RHr\n", uHCReg));
4760
4761 RTGDTR HostGdtr;
4762 PCX86DESCHC pDesc;
4763 ASMGetGDTR(&HostGdtr);
4764 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4765 Log4(("Host CS %#08x\n", u32Val));
4766 if (u32Val < HostGdtr.cbGdt)
4767 {
4768 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4769 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4770 }
4771
4772 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4773 Log4(("Host DS %#08x\n", u32Val));
4774 if (u32Val < HostGdtr.cbGdt)
4775 {
4776 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4777 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4778 }
4779
4780 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4781 Log4(("Host ES %#08x\n", u32Val));
4782 if (u32Val < HostGdtr.cbGdt)
4783 {
4784 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4785 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4786 }
4787
4788 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4789 Log4(("Host FS %#08x\n", u32Val));
4790 if (u32Val < HostGdtr.cbGdt)
4791 {
4792 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4793 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4794 }
4795
4796 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4797 Log4(("Host GS %#08x\n", u32Val));
4798 if (u32Val < HostGdtr.cbGdt)
4799 {
4800 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4801 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4802 }
4803
4804 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4805 Log4(("Host SS %#08x\n", u32Val));
4806 if (u32Val < HostGdtr.cbGdt)
4807 {
4808 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4809 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4810 }
4811
4812 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4813 Log4(("Host TR %#08x\n", u32Val));
4814 if (u32Val < HostGdtr.cbGdt)
4815 {
4816 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4817 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4818 }
4819
4820 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4821 Log4(("Host TR Base %#RHv\n", uHCReg));
4822 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4823 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4824 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4825 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4826 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4827 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4828 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4829 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4830 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4831 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4832 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4833 Log4(("Host RSP %#RHv\n", uHCReg));
4834 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4835 Log4(("Host RIP %#RHv\n", uHCReg));
4836# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4837 if (HMVMX_IS_64BIT_HOST_MODE())
4838 {
4839 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4840 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4841 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4842 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4843 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4844 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4845 }
4846# endif
4847#endif /* VBOX_STRICT */
4848 break;
4849 }
4850
4851 default:
4852 /* Impossible */
4853 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4854 break;
4855 }
4856 NOREF(pVM); NOREF(pCtx);
4857}
4858
4859
4860#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4861#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4862# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4863#endif
4864#ifdef VBOX_STRICT
4865static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4866{
4867 switch (idxField)
4868 {
4869 case VMX_VMCS_GUEST_RIP:
4870 case VMX_VMCS_GUEST_RSP:
4871 case VMX_VMCS_GUEST_SYSENTER_EIP:
4872 case VMX_VMCS_GUEST_SYSENTER_ESP:
4873 case VMX_VMCS_GUEST_GDTR_BASE:
4874 case VMX_VMCS_GUEST_IDTR_BASE:
4875 case VMX_VMCS_GUEST_CS_BASE:
4876 case VMX_VMCS_GUEST_DS_BASE:
4877 case VMX_VMCS_GUEST_ES_BASE:
4878 case VMX_VMCS_GUEST_FS_BASE:
4879 case VMX_VMCS_GUEST_GS_BASE:
4880 case VMX_VMCS_GUEST_SS_BASE:
4881 case VMX_VMCS_GUEST_LDTR_BASE:
4882 case VMX_VMCS_GUEST_TR_BASE:
4883 case VMX_VMCS_GUEST_CR3:
4884 return true;
4885 }
4886 return false;
4887}
4888
4889static bool hmR0VmxIsValidReadField(uint32_t idxField)
4890{
4891 switch (idxField)
4892 {
4893 /* Read-only fields. */
4894 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4895 return true;
4896 }
4897 /* Remaining readable fields should also be writable. */
4898 return hmR0VmxIsValidWriteField(idxField);
4899}
4900#endif /* VBOX_STRICT */
4901
4902
4903/**
4904 * Executes the specified handler in 64-bit mode.
4905 *
4906 * @returns VBox status code.
4907 * @param pVM Pointer to the VM.
4908 * @param pVCpu Pointer to the VMCPU.
4909 * @param pCtx Pointer to the guest CPU context.
4910 * @param enmOp The operation to perform.
4911 * @param cbParam Number of parameters.
4912 * @param paParam Array of 32-bit parameters.
4913 */
4914VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4915 uint32_t *paParam)
4916{
4917 int rc, rc2;
4918 PHMGLOBALCPUINFO pCpu;
4919 RTHCPHYS HCPhysCpuPage;
4920 RTCCUINTREG uOldEflags;
4921
4922 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4923 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4924 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4925 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4926
4927#ifdef VBOX_STRICT
4928 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4929 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4930
4931 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4932 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4933#endif
4934
4935 /* Disable interrupts. */
4936 uOldEflags = ASMIntDisableFlags();
4937
4938#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4939 RTCPUID idHostCpu = RTMpCpuId();
4940 CPUMR0SetLApic(pVCpu, idHostCpu);
4941#endif
4942
4943 pCpu = HMR0GetCurrentCpu();
4944 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4945
4946 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4947 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4948
4949 /* Leave VMX Root Mode. */
4950 VMXDisable();
4951
4952 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4953
4954 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4955 CPUMSetHyperEIP(pVCpu, enmOp);
4956 for (int i = (int)cbParam - 1; i >= 0; i--)
4957 CPUMPushHyper(pVCpu, paParam[i]);
4958
4959 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4960
4961 /* Call the switcher. */
4962 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4963 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4964
4965 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4966 /* Make sure the VMX instructions don't cause #UD faults. */
4967 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4968
4969 /* Re-enter VMX Root Mode */
4970 rc2 = VMXEnable(HCPhysCpuPage);
4971 if (RT_FAILURE(rc2))
4972 {
4973 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4974 ASMSetFlags(uOldEflags);
4975 return rc2;
4976 }
4977
4978 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4979 AssertRC(rc2);
4980 Assert(!(ASMGetFlags() & X86_EFL_IF));
4981 ASMSetFlags(uOldEflags);
4982 return rc;
4983}
4984
4985
4986/**
4987 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4988 * supporting 64-bit guests.
4989 *
4990 * @returns VBox status code.
4991 * @param fResume Whether to VMLAUNCH or VMRESUME.
4992 * @param pCtx Pointer to the guest-CPU context.
4993 * @param pCache Pointer to the VMCS cache.
4994 * @param pVM Pointer to the VM.
4995 * @param pVCpu Pointer to the VMCPU.
4996 */
4997DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4998{
4999 uint32_t aParam[6];
5000 PHMGLOBALCPUINFO pCpu = NULL;
5001 RTHCPHYS HCPhysCpuPage = 0;
5002 int rc = VERR_INTERNAL_ERROR_5;
5003
5004 pCpu = HMR0GetCurrentCpu();
5005 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5006
5007#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5008 pCache->uPos = 1;
5009 pCache->interPD = PGMGetInterPaeCR3(pVM);
5010 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5011#endif
5012
5013#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5014 pCache->TestIn.HCPhysCpuPage = 0;
5015 pCache->TestIn.HCPhysVmcs = 0;
5016 pCache->TestIn.pCache = 0;
5017 pCache->TestOut.HCPhysVmcs = 0;
5018 pCache->TestOut.pCache = 0;
5019 pCache->TestOut.pCtx = 0;
5020 pCache->TestOut.eflags = 0;
5021#endif
5022
5023 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5024 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5025 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5026 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5027 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5028 aParam[5] = 0;
5029
5030#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5031 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5032 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5033#endif
5034 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5035
5036#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5037 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5038 Assert(pCtx->dr[4] == 10);
5039 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5040#endif
5041
5042#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5043 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5044 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5045 pVCpu->hm.s.vmx.HCPhysVmcs));
5046 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5047 pCache->TestOut.HCPhysVmcs));
5048 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5049 pCache->TestOut.pCache));
5050 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5051 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5052 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5053 pCache->TestOut.pCtx));
5054 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5055#endif
5056 return rc;
5057}
5058
5059
5060/**
5061 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5062 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5063 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5064 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5065 *
5066 * @returns VBox status code.
5067 * @param pVM Pointer to the VM.
5068 * @param pVCpu Pointer to the VMCPU.
5069 */
5070static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5071{
5072#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5073{ \
5074 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5075 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5076 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5077 ++cReadFields; \
5078}
5079
5080 AssertPtr(pVM);
5081 AssertPtr(pVCpu);
5082 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5083 uint32_t cReadFields = 0;
5084
5085 /*
5086 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5087 * and serve to indicate exceptions to the rules.
5088 */
5089
5090 /* Guest-natural selector base fields. */
5091#if 0
5092 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5093 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5094 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5095#endif
5096 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5097 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5098 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5099 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5100 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5101 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5102 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5103 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5104 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5105 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5106 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5107 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5108#if 0
5109 /* Unused natural width guest-state fields. */
5110 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5111 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5112#endif
5113 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5114 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5115
5116 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5117#if 0
5118 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5119 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5120 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5121 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5122 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5123 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5124 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5125 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5126 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5127#endif
5128
5129 /* Natural width guest-state fields. */
5130 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5131#if 0
5132 /* Currently unused field. */
5133 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5134#endif
5135
5136 if (pVM->hm.s.fNestedPaging)
5137 {
5138 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5139 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5140 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5141 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5142 }
5143 else
5144 {
5145 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5146 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5147 }
5148
5149#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5150 return VINF_SUCCESS;
5151}
5152
5153
5154/**
5155 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5156 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5157 * darwin, running 64-bit guests).
5158 *
5159 * @returns VBox status code.
5160 * @param pVCpu Pointer to the VMCPU.
5161 * @param idxField The VMCS field encoding.
5162 * @param u64Val 16, 32 or 64-bit value.
5163 */
5164VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5165{
5166 int rc;
5167 switch (idxField)
5168 {
5169 /*
5170 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5171 */
5172 /* 64-bit Control fields. */
5173 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5174 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5175 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5176 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5177 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5178 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5179 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5180 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5181 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5182 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5183 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5184 case VMX_VMCS64_CTRL_EPTP_FULL:
5185 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5186 /* 64-bit Guest-state fields. */
5187 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5188 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5189 case VMX_VMCS64_GUEST_PAT_FULL:
5190 case VMX_VMCS64_GUEST_EFER_FULL:
5191 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5192 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5193 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5194 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5195 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5196 /* 64-bit Host-state fields. */
5197 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5198 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5199 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5200 {
5201 rc = VMXWriteVmcs32(idxField, u64Val);
5202 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5203 break;
5204 }
5205
5206 /*
5207 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5208 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5209 */
5210 /* Natural-width Guest-state fields. */
5211 case VMX_VMCS_GUEST_CR3:
5212 case VMX_VMCS_GUEST_ES_BASE:
5213 case VMX_VMCS_GUEST_CS_BASE:
5214 case VMX_VMCS_GUEST_SS_BASE:
5215 case VMX_VMCS_GUEST_DS_BASE:
5216 case VMX_VMCS_GUEST_FS_BASE:
5217 case VMX_VMCS_GUEST_GS_BASE:
5218 case VMX_VMCS_GUEST_LDTR_BASE:
5219 case VMX_VMCS_GUEST_TR_BASE:
5220 case VMX_VMCS_GUEST_GDTR_BASE:
5221 case VMX_VMCS_GUEST_IDTR_BASE:
5222 case VMX_VMCS_GUEST_RSP:
5223 case VMX_VMCS_GUEST_RIP:
5224 case VMX_VMCS_GUEST_SYSENTER_ESP:
5225 case VMX_VMCS_GUEST_SYSENTER_EIP:
5226 {
5227 if (!(u64Val >> 32))
5228 {
5229 /* If this field is 64-bit, VT-x will zero out the top bits. */
5230 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5231 }
5232 else
5233 {
5234 /* Assert that only the 32->64 switcher case should ever come here. */
5235 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5236 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5237 }
5238 break;
5239 }
5240
5241 default:
5242 {
5243 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5244 rc = VERR_INVALID_PARAMETER;
5245 break;
5246 }
5247 }
5248 AssertRCReturn(rc, rc);
5249 return rc;
5250}
5251
5252
5253/**
5254 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5255 * hosts (except darwin) for 64-bit guests.
5256 *
5257 * @param pVCpu Pointer to the VMCPU.
5258 * @param idxField The VMCS field encoding.
5259 * @param u64Val 16, 32 or 64-bit value.
5260 */
5261VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5262{
5263 AssertPtr(pVCpu);
5264 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5265
5266 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5267 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5268
5269 /* Make sure there are no duplicates. */
5270 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5271 {
5272 if (pCache->Write.aField[i] == idxField)
5273 {
5274 pCache->Write.aFieldVal[i] = u64Val;
5275 return VINF_SUCCESS;
5276 }
5277 }
5278
5279 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5280 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5281 pCache->Write.cValidEntries++;
5282 return VINF_SUCCESS;
5283}
5284
5285/* Enable later when the assembly code uses these as callbacks. */
5286#if 0
5287/*
5288 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5289 *
5290 * @param pVCpu Pointer to the VMCPU.
5291 * @param pCache Pointer to the VMCS cache.
5292 *
5293 * @remarks No-long-jump zone!!!
5294 */
5295VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5296{
5297 AssertPtr(pCache);
5298 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5299 {
5300 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5301 AssertRC(rc);
5302 }
5303 pCache->Write.cValidEntries = 0;
5304}
5305
5306
5307/**
5308 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5309 *
5310 * @param pVCpu Pointer to the VMCPU.
5311 * @param pCache Pointer to the VMCS cache.
5312 *
5313 * @remarks No-long-jump zone!!!
5314 */
5315VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5316{
5317 AssertPtr(pCache);
5318 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5319 {
5320 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5321 AssertRC(rc);
5322 }
5323}
5324#endif
5325#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5326
5327
5328/**
5329 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5330 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5331 * timer.
5332 *
5333 * @returns VBox status code.
5334 * @param pVCpu Pointer to the VMCPU.
5335 *
5336 * @remarks No-long-jump zone!!!
5337 */
5338static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5339{
5340 int rc = VERR_INTERNAL_ERROR_5;
5341 bool fOffsettedTsc = false;
5342 PVM pVM = pVCpu->CTX_SUFF(pVM);
5343 if (pVM->hm.s.vmx.fUsePreemptTimer)
5344 {
5345 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5346
5347 /* Make sure the returned values have sane upper and lower boundaries. */
5348 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5349 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5350 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5351 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5352
5353 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5354 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5355 }
5356 else
5357 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5358
5359 if (fOffsettedTsc)
5360 {
5361 uint64_t u64CurTSC = ASMReadTSC();
5362 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5363 {
5364 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5365 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5366
5367 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5368 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5369 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5370 }
5371 else
5372 {
5373 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5374 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5375 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5376 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5377 }
5378 }
5379 else
5380 {
5381 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5382 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5383 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5384 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5385 }
5386}
5387
5388
5389/**
5390 * Determines if an exception is a contributory exception. Contributory
5391 * exceptions are ones which can cause double-faults. Page-fault is
5392 * intentionally not included here as it's a conditional contributory exception.
5393 *
5394 * @returns true if the exception is contributory, false otherwise.
5395 * @param uVector The exception vector.
5396 */
5397DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5398{
5399 switch (uVector)
5400 {
5401 case X86_XCPT_GP:
5402 case X86_XCPT_SS:
5403 case X86_XCPT_NP:
5404 case X86_XCPT_TS:
5405 case X86_XCPT_DE:
5406 return true;
5407 default:
5408 break;
5409 }
5410 return false;
5411}
5412
5413
5414/**
5415 * Sets an event as a pending event to be injected into the guest.
5416 *
5417 * @param pVCpu Pointer to the VMCPU.
5418 * @param u32IntInfo The VM-entry interruption-information field.
5419 * @param cbInstr The VM-entry instruction length in bytes (for software
5420 * interrupts, exceptions and privileged software
5421 * exceptions).
5422 * @param u32ErrCode The VM-entry exception error code.
5423 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5424 * page-fault.
5425 *
5426 * @remarks Statistics counter assumes this is a guest event being injected or
5427 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5428 * always incremented.
5429 */
5430DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5431 RTGCUINTPTR GCPtrFaultAddress)
5432{
5433 Assert(!pVCpu->hm.s.Event.fPending);
5434 pVCpu->hm.s.Event.fPending = true;
5435 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5436 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5437 pVCpu->hm.s.Event.cbInstr = cbInstr;
5438 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5439
5440 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5441}
5442
5443
5444/**
5445 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5446 *
5447 * @param pVCpu Pointer to the VMCPU.
5448 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5449 * out-of-sync. Make sure to update the required fields
5450 * before using them.
5451 */
5452DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5453{
5454 NOREF(pMixedCtx);
5455 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5456 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5457 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5458 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5459}
5460
5461
5462/**
5463 * Handle a condition that occurred while delivering an event through the guest
5464 * IDT.
5465 *
5466 * @returns VBox status code (informational error codes included).
5467 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5468 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5469 * continue execution of the guest which will delivery the #DF.
5470 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5471 *
5472 * @param pVCpu Pointer to the VMCPU.
5473 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5474 * out-of-sync. Make sure to update the required fields
5475 * before using them.
5476 * @param pVmxTransient Pointer to the VMX transient structure.
5477 *
5478 * @remarks No-long-jump zone!!!
5479 */
5480static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5481{
5482 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5483 AssertRCReturn(rc, rc);
5484 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5485 {
5486 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5487 AssertRCReturn(rc, rc);
5488
5489 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5490 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5491 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5492
5493 typedef enum
5494 {
5495 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5496 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5497 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5498 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5499 } VMXREFLECTXCPT;
5500
5501 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5502 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5503 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5504 {
5505 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5506 {
5507 enmReflect = VMXREFLECTXCPT_XCPT;
5508#ifdef VBOX_STRICT
5509 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5510 && uExitVector == X86_XCPT_PF)
5511 {
5512 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5513 }
5514#endif
5515 if ( uExitVector == X86_XCPT_PF
5516 && uIdtVector == X86_XCPT_PF)
5517 {
5518 pVmxTransient->fVectoringPF = true;
5519 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5520 }
5521 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5522 && hmR0VmxIsContributoryXcpt(uExitVector)
5523 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5524 || uIdtVector == X86_XCPT_PF))
5525 {
5526 enmReflect = VMXREFLECTXCPT_DF;
5527 }
5528 else if (uIdtVector == X86_XCPT_DF)
5529 enmReflect = VMXREFLECTXCPT_TF;
5530 }
5531 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5532 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5533 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5534 {
5535 /*
5536 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5537 * (whatever they are) as they reoccur when restarting the instruction.
5538 */
5539 enmReflect = VMXREFLECTXCPT_XCPT;
5540 }
5541 }
5542 else
5543 {
5544 /*
5545 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5546 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5547 * original exception to the guest after handling the VM-exit.
5548 */
5549 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5550 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5551 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5552 {
5553 enmReflect = VMXREFLECTXCPT_XCPT;
5554 }
5555 }
5556
5557 switch (enmReflect)
5558 {
5559 case VMXREFLECTXCPT_XCPT:
5560 {
5561 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5562 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5563 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5564
5565 uint32_t u32ErrCode = 0;
5566 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5567 {
5568 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5569 AssertRCReturn(rc, rc);
5570 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5571 }
5572
5573 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5574 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5575 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5576 rc = VINF_SUCCESS;
5577 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5578 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5579
5580 break;
5581 }
5582
5583 case VMXREFLECTXCPT_DF:
5584 {
5585 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5586 rc = VINF_HM_DOUBLE_FAULT;
5587 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5588 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5589
5590 break;
5591 }
5592
5593 case VMXREFLECTXCPT_TF:
5594 {
5595 rc = VINF_EM_RESET;
5596 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5597 uExitVector));
5598 break;
5599 }
5600
5601 default:
5602 Assert(rc == VINF_SUCCESS);
5603 break;
5604 }
5605 }
5606 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5607 return rc;
5608}
5609
5610
5611/**
5612 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5613 *
5614 * @returns VBox status code.
5615 * @param pVCpu Pointer to the VMCPU.
5616 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5617 * out-of-sync. Make sure to update the required fields
5618 * before using them.
5619 *
5620 * @remarks No-long-jump zone!!!
5621 */
5622static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5623{
5624 NOREF(pMixedCtx);
5625
5626 /*
5627 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5628 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5629 */
5630 VMMRZCallRing3Disable(pVCpu);
5631 HM_DISABLE_PREEMPT_IF_NEEDED();
5632
5633 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5634 {
5635 uint32_t uVal = 0;
5636 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5637 AssertRCReturn(rc, rc);
5638
5639 uint32_t uShadow = 0;
5640 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5641 AssertRCReturn(rc, rc);
5642
5643 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5644 CPUMSetGuestCR0(pVCpu, uVal);
5645 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5646 }
5647
5648 HM_RESTORE_PREEMPT_IF_NEEDED();
5649 VMMRZCallRing3Enable(pVCpu);
5650 return VINF_SUCCESS;
5651}
5652
5653
5654/**
5655 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5656 *
5657 * @returns VBox status code.
5658 * @param pVCpu Pointer to the VMCPU.
5659 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5660 * out-of-sync. Make sure to update the required fields
5661 * before using them.
5662 *
5663 * @remarks No-long-jump zone!!!
5664 */
5665static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5666{
5667 NOREF(pMixedCtx);
5668
5669 int rc = VINF_SUCCESS;
5670 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5671 {
5672 uint32_t uVal = 0;
5673 uint32_t uShadow = 0;
5674 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5675 AssertRCReturn(rc, rc);
5676 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5677 AssertRCReturn(rc, rc);
5678
5679 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5680 CPUMSetGuestCR4(pVCpu, uVal);
5681 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5682 }
5683 return rc;
5684}
5685
5686
5687/**
5688 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5689 *
5690 * @returns VBox status code.
5691 * @param pVCpu Pointer to the VMCPU.
5692 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5693 * out-of-sync. Make sure to update the required fields
5694 * before using them.
5695 *
5696 * @remarks No-long-jump zone!!!
5697 */
5698static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5699{
5700 int rc = VINF_SUCCESS;
5701 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5702 {
5703 uint64_t u64Val = 0;
5704 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5705 AssertRCReturn(rc, rc);
5706
5707 pMixedCtx->rip = u64Val;
5708 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5709 }
5710 return rc;
5711}
5712
5713
5714/**
5715 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5716 *
5717 * @returns VBox status code.
5718 * @param pVCpu Pointer to the VMCPU.
5719 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5720 * out-of-sync. Make sure to update the required fields
5721 * before using them.
5722 *
5723 * @remarks No-long-jump zone!!!
5724 */
5725static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5726{
5727 int rc = VINF_SUCCESS;
5728 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5729 {
5730 uint64_t u64Val = 0;
5731 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5732 AssertRCReturn(rc, rc);
5733
5734 pMixedCtx->rsp = u64Val;
5735 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5736 }
5737 return rc;
5738}
5739
5740
5741/**
5742 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5743 *
5744 * @returns VBox status code.
5745 * @param pVCpu Pointer to the VMCPU.
5746 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747 * out-of-sync. Make sure to update the required fields
5748 * before using them.
5749 *
5750 * @remarks No-long-jump zone!!!
5751 */
5752static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753{
5754 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5755 {
5756 uint32_t uVal = 0;
5757 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5758 AssertRCReturn(rc, rc);
5759
5760 pMixedCtx->eflags.u32 = uVal;
5761 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5762 {
5763 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5764 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5765
5766 pMixedCtx->eflags.Bits.u1VM = 0;
5767 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5768 }
5769
5770 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5771 }
5772 return VINF_SUCCESS;
5773}
5774
5775
5776/**
5777 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5778 * guest-CPU context.
5779 */
5780DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5781{
5782 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5783 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5784 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5785 return rc;
5786}
5787
5788
5789/**
5790 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5791 * from the guest-state area in the VMCS.
5792 *
5793 * @param pVCpu Pointer to the VMCPU.
5794 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5795 * out-of-sync. Make sure to update the required fields
5796 * before using them.
5797 *
5798 * @remarks No-long-jump zone!!!
5799 */
5800static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5801{
5802 uint32_t uIntrState = 0;
5803 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5804 AssertRC(rc);
5805
5806 if (!uIntrState)
5807 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5808 else
5809 {
5810 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5811 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5812 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5813 AssertRC(rc);
5814 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5815 AssertRC(rc);
5816
5817 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5818 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5819 }
5820}
5821
5822
5823/**
5824 * Saves the guest's activity state.
5825 *
5826 * @returns VBox status code.
5827 * @param pVCpu Pointer to the VMCPU.
5828 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5829 * out-of-sync. Make sure to update the required fields
5830 * before using them.
5831 *
5832 * @remarks No-long-jump zone!!!
5833 */
5834static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5835{
5836 NOREF(pMixedCtx);
5837 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5838 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5839 return VINF_SUCCESS;
5840}
5841
5842
5843/**
5844 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5845 * the current VMCS into the guest-CPU context.
5846 *
5847 * @returns VBox status code.
5848 * @param pVCpu Pointer to the VMCPU.
5849 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5850 * out-of-sync. Make sure to update the required fields
5851 * before using them.
5852 *
5853 * @remarks No-long-jump zone!!!
5854 */
5855static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5856{
5857 int rc = VINF_SUCCESS;
5858 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5859 {
5860 uint32_t u32Val = 0;
5861 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5862 pMixedCtx->SysEnter.cs = u32Val;
5863 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5864 }
5865
5866 uint64_t u64Val = 0;
5867 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5868 {
5869 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5870 pMixedCtx->SysEnter.eip = u64Val;
5871 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5872 }
5873 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5874 {
5875 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5876 pMixedCtx->SysEnter.esp = u64Val;
5877 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5878 }
5879 return rc;
5880}
5881
5882
5883/**
5884 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5885 * the CPU back into the guest-CPU context.
5886 *
5887 * @returns VBox status code.
5888 * @param pVCpu Pointer to the VMCPU.
5889 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5890 * out-of-sync. Make sure to update the required fields
5891 * before using them.
5892 *
5893 * @remarks No-long-jump zone!!!
5894 */
5895static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5896{
5897#if HC_ARCH_BITS == 64
5898 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5899 {
5900 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5901 VMMRZCallRing3Disable(pVCpu);
5902 HM_DISABLE_PREEMPT_IF_NEEDED();
5903
5904 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5905 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5906 {
5907 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5908 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5909 }
5910
5911 HM_RESTORE_PREEMPT_IF_NEEDED();
5912 VMMRZCallRing3Enable(pVCpu);
5913 }
5914 else
5915 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5916#else
5917 NOREF(pMixedCtx);
5918 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5919#endif
5920
5921 return VINF_SUCCESS;
5922}
5923
5924
5925/**
5926 * Saves the auto load/store'd guest MSRs from the current VMCS into
5927 * the guest-CPU context.
5928 *
5929 * @returns VBox status code.
5930 * @param pVCpu Pointer to the VMCPU.
5931 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5932 * out-of-sync. Make sure to update the required fields
5933 * before using them.
5934 *
5935 * @remarks No-long-jump zone!!!
5936 */
5937static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5938{
5939 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5940 return VINF_SUCCESS;
5941
5942 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5943 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5944 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5945 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5946 {
5947 switch (pMsr->u32Msr)
5948 {
5949 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5950 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5951 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5952 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5953 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5954 default:
5955 {
5956 AssertFailed();
5957 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5958 }
5959 }
5960 }
5961
5962 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5963 return VINF_SUCCESS;
5964}
5965
5966
5967/**
5968 * Saves the guest control registers from the current VMCS into the guest-CPU
5969 * context.
5970 *
5971 * @returns VBox status code.
5972 * @param pVCpu Pointer to the VMCPU.
5973 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5974 * out-of-sync. Make sure to update the required fields
5975 * before using them.
5976 *
5977 * @remarks No-long-jump zone!!!
5978 */
5979static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5980{
5981 /* Guest CR0. Guest FPU. */
5982 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5983 AssertRCReturn(rc, rc);
5984
5985 /* Guest CR4. */
5986 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5987 AssertRCReturn(rc, rc);
5988
5989 /* Guest CR2 - updated always during the world-switch or in #PF. */
5990 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5991 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5992 {
5993 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5994 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5995
5996 PVM pVM = pVCpu->CTX_SUFF(pVM);
5997 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5998 || ( pVM->hm.s.fNestedPaging
5999 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6000 {
6001 uint64_t u64Val = 0;
6002 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6003 if (pMixedCtx->cr3 != u64Val)
6004 {
6005 CPUMSetGuestCR3(pVCpu, u64Val);
6006 if (VMMRZCallRing3IsEnabled(pVCpu))
6007 {
6008 PGMUpdateCR3(pVCpu, u64Val);
6009 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6010 }
6011 else
6012 {
6013 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6014 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6015 }
6016 }
6017
6018 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6019 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6020 {
6021 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6022 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6023 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6024 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6025
6026 if (VMMRZCallRing3IsEnabled(pVCpu))
6027 {
6028 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6029 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6030 }
6031 else
6032 {
6033 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6034 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6035 }
6036 }
6037 }
6038
6039 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6040 }
6041
6042 /*
6043 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6044 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6045 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6046 *
6047 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6048 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6049 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6050 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6051 *
6052 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6053 */
6054 if (VMMRZCallRing3IsEnabled(pVCpu))
6055 {
6056 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6057 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6058
6059 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6060 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6061
6062 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6063 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6064 }
6065
6066 return rc;
6067}
6068
6069
6070/**
6071 * Reads a guest segment register from the current VMCS into the guest-CPU
6072 * context.
6073 *
6074 * @returns VBox status code.
6075 * @param pVCpu Pointer to the VMCPU.
6076 * @param idxSel Index of the selector in the VMCS.
6077 * @param idxLimit Index of the segment limit in the VMCS.
6078 * @param idxBase Index of the segment base in the VMCS.
6079 * @param idxAccess Index of the access rights of the segment in the VMCS.
6080 * @param pSelReg Pointer to the segment selector.
6081 *
6082 * @remarks No-long-jump zone!!!
6083 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6084 * macro as that takes care of whether to read from the VMCS cache or
6085 * not.
6086 */
6087DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6088 PCPUMSELREG pSelReg)
6089{
6090 NOREF(pVCpu);
6091
6092 uint32_t u32Val = 0;
6093 int rc = VMXReadVmcs32(idxSel, &u32Val);
6094 AssertRCReturn(rc, rc);
6095 pSelReg->Sel = (uint16_t)u32Val;
6096 pSelReg->ValidSel = (uint16_t)u32Val;
6097 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6098
6099 rc = VMXReadVmcs32(idxLimit, &u32Val);
6100 AssertRCReturn(rc, rc);
6101 pSelReg->u32Limit = u32Val;
6102
6103 uint64_t u64Val = 0;
6104 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6105 AssertRCReturn(rc, rc);
6106 pSelReg->u64Base = u64Val;
6107
6108 rc = VMXReadVmcs32(idxAccess, &u32Val);
6109 AssertRCReturn(rc, rc);
6110 pSelReg->Attr.u = u32Val;
6111
6112 /*
6113 * If VT-x marks the segment as unusable, most other bits remain undefined:
6114 * - For CS the L, D and G bits have meaning.
6115 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6116 * - For the remaining data segments no bits are defined.
6117 *
6118 * The present bit and the unusable bit has been observed to be set at the
6119 * same time (the selector was supposed to invalid as we started executing
6120 * a V8086 interrupt in ring-0).
6121 *
6122 * What should be important for the rest of the VBox code, is that the P bit is
6123 * cleared. Some of the other VBox code recognizes the unusable bit, but
6124 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6125 * safe side here, we'll strip off P and other bits we don't care about. If
6126 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6127 *
6128 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6129 */
6130 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6131 {
6132 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6133
6134 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6135 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6136 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6137
6138 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6139#ifdef DEBUG_bird
6140 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6141 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6142 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6143#endif
6144 }
6145 return VINF_SUCCESS;
6146}
6147
6148
6149#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6150# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6151 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6152 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6153#else
6154# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6155 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6156 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6157#endif
6158
6159
6160/**
6161 * Saves the guest segment registers from the current VMCS into the guest-CPU
6162 * context.
6163 *
6164 * @returns VBox status code.
6165 * @param pVCpu Pointer to the VMCPU.
6166 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6167 * out-of-sync. Make sure to update the required fields
6168 * before using them.
6169 *
6170 * @remarks No-long-jump zone!!!
6171 */
6172static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6173{
6174 /* Guest segment registers. */
6175 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6176 {
6177 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6178 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6179 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6180 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6181 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6182 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6183 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6184
6185 /* Restore segment attributes for real-on-v86 mode hack. */
6186 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6187 {
6188 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6189 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6190 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6191 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6192 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6193 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6194 }
6195 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6196 }
6197
6198 return VINF_SUCCESS;
6199}
6200
6201
6202/**
6203 * Saves the guest descriptor table registers and task register from the current
6204 * VMCS into the guest-CPU context.
6205 *
6206 * @returns VBox status code.
6207 * @param pVCpu Pointer to the VMCPU.
6208 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6209 * out-of-sync. Make sure to update the required fields
6210 * before using them.
6211 *
6212 * @remarks No-long-jump zone!!!
6213 */
6214static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6215{
6216 int rc = VINF_SUCCESS;
6217
6218 /* Guest LDTR. */
6219 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6220 {
6221 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6222 AssertRCReturn(rc, rc);
6223 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6224 }
6225
6226 /* Guest GDTR. */
6227 uint64_t u64Val = 0;
6228 uint32_t u32Val = 0;
6229 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6230 {
6231 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6232 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6233 pMixedCtx->gdtr.pGdt = u64Val;
6234 pMixedCtx->gdtr.cbGdt = u32Val;
6235 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6236 }
6237
6238 /* Guest IDTR. */
6239 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6240 {
6241 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6242 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6243 pMixedCtx->idtr.pIdt = u64Val;
6244 pMixedCtx->idtr.cbIdt = u32Val;
6245 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6246 }
6247
6248 /* Guest TR. */
6249 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6250 {
6251 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6252 AssertRCReturn(rc, rc);
6253
6254 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6255 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6256 {
6257 rc = VMXLOCAL_READ_SEG(TR, tr);
6258 AssertRCReturn(rc, rc);
6259 }
6260 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6261 }
6262 return rc;
6263}
6264
6265#undef VMXLOCAL_READ_SEG
6266
6267
6268/**
6269 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6270 * context.
6271 *
6272 * @returns VBox status code.
6273 * @param pVCpu Pointer to the VMCPU.
6274 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6275 * out-of-sync. Make sure to update the required fields
6276 * before using them.
6277 *
6278 * @remarks No-long-jump zone!!!
6279 */
6280static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6281{
6282 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6283 {
6284 if (!pVCpu->hm.s.fUsingHyperDR7)
6285 {
6286 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6287 uint32_t u32Val;
6288 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6289 pMixedCtx->dr[7] = u32Val;
6290 }
6291
6292 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6293 }
6294 return VINF_SUCCESS;
6295}
6296
6297
6298/**
6299 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6300 *
6301 * @returns VBox status code.
6302 * @param pVCpu Pointer to the VMCPU.
6303 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6304 * out-of-sync. Make sure to update the required fields
6305 * before using them.
6306 *
6307 * @remarks No-long-jump zone!!!
6308 */
6309static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6310{
6311 NOREF(pMixedCtx);
6312
6313 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6314 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6315 return VINF_SUCCESS;
6316}
6317
6318
6319/**
6320 * Saves the entire guest state from the currently active VMCS into the
6321 * guest-CPU context. This essentially VMREADs all guest-data.
6322 *
6323 * @returns VBox status code.
6324 * @param pVCpu Pointer to the VMCPU.
6325 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6326 * out-of-sync. Make sure to update the required fields
6327 * before using them.
6328 */
6329static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6330{
6331 Assert(pVCpu);
6332 Assert(pMixedCtx);
6333
6334 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6335 return VINF_SUCCESS;
6336
6337 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6338 again on the ring-3 callback path, there is no real need to. */
6339 if (VMMRZCallRing3IsEnabled(pVCpu))
6340 VMMR0LogFlushDisable(pVCpu);
6341 else
6342 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6343 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6344
6345 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6346 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6347
6348 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6349 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6350
6351 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6352 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6353
6354 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6355 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6356
6357 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6358 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6359
6360 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6361 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6362
6363 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6364 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6365
6366 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6367 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6368
6369 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6370 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6371
6372 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6373 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6374
6375 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6376 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6377
6378 if (VMMRZCallRing3IsEnabled(pVCpu))
6379 VMMR0LogFlushEnable(pVCpu);
6380
6381 return rc;
6382}
6383
6384
6385/**
6386 * Check per-VM and per-VCPU force flag actions that require us to go back to
6387 * ring-3 for one reason or another.
6388 *
6389 * @returns VBox status code (information status code included).
6390 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6391 * ring-3.
6392 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6393 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6394 * interrupts)
6395 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6396 * all EMTs to be in ring-3.
6397 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6398 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6399 * to the EM loop.
6400 *
6401 * @param pVM Pointer to the VM.
6402 * @param pVCpu Pointer to the VMCPU.
6403 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6404 * out-of-sync. Make sure to update the required fields
6405 * before using them.
6406 */
6407static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6408{
6409 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6410
6411 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6412 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6413 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6414 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6415 {
6416 /* We need the control registers now, make sure the guest-CPU context is updated. */
6417 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6418 AssertRCReturn(rc3, rc3);
6419
6420 /* Pending HM CR3 sync. */
6421 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6422 {
6423 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6424 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6425 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6426 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6427 }
6428
6429 /* Pending HM PAE PDPEs. */
6430 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6431 {
6432 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6433 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6434 }
6435
6436 /* Pending PGM C3 sync. */
6437 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6438 {
6439 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6440 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6441 if (rc2 != VINF_SUCCESS)
6442 {
6443 AssertRC(rc2);
6444 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6445 return rc2;
6446 }
6447 }
6448
6449 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6450 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6451 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6452 {
6453 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6454 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6455 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6456 return rc2;
6457 }
6458
6459 /* Pending VM request packets, such as hardware interrupts. */
6460 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6461 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6462 {
6463 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6464 return VINF_EM_PENDING_REQUEST;
6465 }
6466
6467 /* Pending PGM pool flushes. */
6468 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6469 {
6470 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6471 return VINF_PGM_POOL_FLUSH_PENDING;
6472 }
6473
6474 /* Pending DMA requests. */
6475 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6476 {
6477 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6478 return VINF_EM_RAW_TO_R3;
6479 }
6480 }
6481
6482 return VINF_SUCCESS;
6483}
6484
6485
6486/**
6487 * Converts any TRPM trap into a pending HM event. This is typically used when
6488 * entering from ring-3 (not longjmp returns).
6489 *
6490 * @param pVCpu Pointer to the VMCPU.
6491 */
6492static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6493{
6494 Assert(TRPMHasTrap(pVCpu));
6495 Assert(!pVCpu->hm.s.Event.fPending);
6496
6497 uint8_t uVector;
6498 TRPMEVENT enmTrpmEvent;
6499 RTGCUINT uErrCode;
6500 RTGCUINTPTR GCPtrFaultAddress;
6501 uint8_t cbInstr;
6502
6503 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6504 AssertRC(rc);
6505
6506 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6507 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6508 if (enmTrpmEvent == TRPM_TRAP)
6509 {
6510 switch (uVector)
6511 {
6512 case X86_XCPT_BP:
6513 case X86_XCPT_OF:
6514 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6515 break;
6516
6517 case X86_XCPT_PF:
6518 case X86_XCPT_DF:
6519 case X86_XCPT_TS:
6520 case X86_XCPT_NP:
6521 case X86_XCPT_SS:
6522 case X86_XCPT_GP:
6523 case X86_XCPT_AC:
6524 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6525 /* no break! */
6526 default:
6527 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6528 break;
6529 }
6530 }
6531 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6532 {
6533 if (uVector == X86_XCPT_NMI)
6534 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6535 else
6536 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6537 }
6538 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6539 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6540 else
6541 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6542
6543 rc = TRPMResetTrap(pVCpu);
6544 AssertRC(rc);
6545 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6546 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6547
6548 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6549 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6550}
6551
6552
6553/**
6554 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6555 * VT-x to execute any instruction.
6556 *
6557 * @param pvCpu Pointer to the VMCPU.
6558 */
6559static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6560{
6561 Assert(pVCpu->hm.s.Event.fPending);
6562
6563 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6564 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6565 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6566 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6567
6568 /* If a trap was already pending, we did something wrong! */
6569 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6570
6571 TRPMEVENT enmTrapType;
6572 switch (uVectorType)
6573 {
6574 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6575 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6576 enmTrapType = TRPM_HARDWARE_INT;
6577 break;
6578
6579 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6580 enmTrapType = TRPM_SOFTWARE_INT;
6581 break;
6582
6583 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6584 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6585 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6586 enmTrapType = TRPM_TRAP;
6587 break;
6588
6589 default:
6590 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6591 enmTrapType = TRPM_32BIT_HACK;
6592 break;
6593 }
6594
6595 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6596
6597 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6598 AssertRC(rc);
6599
6600 if (fErrorCodeValid)
6601 TRPMSetErrorCode(pVCpu, uErrorCode);
6602
6603 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6604 && uVector == X86_XCPT_PF)
6605 {
6606 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6607 }
6608 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6609 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6610 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6611 {
6612 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6613 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6614 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6615 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6616 }
6617 pVCpu->hm.s.Event.fPending = false;
6618}
6619
6620
6621/**
6622 * Does the necessary state syncing before returning to ring-3 for any reason
6623 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6624 *
6625 * @returns VBox status code.
6626 * @param pVM Pointer to the VM.
6627 * @param pVCpu Pointer to the VMCPU.
6628 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6629 * be out-of-sync. Make sure to update the required
6630 * fields before using them.
6631 * @param fSaveGuestState Whether to save the guest state or not.
6632 *
6633 * @remarks No-long-jmp zone!!!
6634 */
6635static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6636{
6637 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6638 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6639
6640 RTCPUID idCpu = RTMpCpuId();
6641 Log4Func(("HostCpuId=%u\n", idCpu));
6642
6643 /*
6644 * !!! IMPORTANT !!!
6645 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6646 */
6647
6648 /* Save the guest state if necessary. */
6649 if ( fSaveGuestState
6650 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6651 {
6652 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6653 AssertRCReturn(rc, rc);
6654 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6655 }
6656
6657 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6658 if (CPUMIsGuestFPUStateActive(pVCpu))
6659 {
6660 /* We shouldn't reload CR0 without saving it first. */
6661 if (!fSaveGuestState)
6662 {
6663 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6664 AssertRCReturn(rc, rc);
6665 }
6666 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6667 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6668 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6669 }
6670
6671 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6672#ifdef VBOX_STRICT
6673 if (CPUMIsHyperDebugStateActive(pVCpu))
6674 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6675#endif
6676 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6677 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6678 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6679 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6680
6681#if HC_ARCH_BITS == 64
6682 /* Restore host-state bits that VT-x only restores partially. */
6683 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6684 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6685 {
6686 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6687 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6688 }
6689 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6690#endif
6691
6692#if HC_ARCH_BITS == 64
6693 /* Restore the host MSRs as we're leaving VT-x context. */
6694 if ( pVM->hm.s.fAllow64BitGuests
6695 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6696 {
6697 /* We shouldn't reload the guest MSRs without saving it first. */
6698 if (!fSaveGuestState)
6699 {
6700 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6701 AssertRCReturn(rc, rc);
6702 }
6703 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6704 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6705 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6706 }
6707#endif
6708
6709 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6710 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6711
6712 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6713 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6714 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6715 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6716 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6717 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6718 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6719 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6720
6721 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6722
6723 /** @todo This partially defeats the purpose of having preemption hooks.
6724 * The problem is, deregistering the hooks should be moved to a place that
6725 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6726 * context.
6727 */
6728 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6729 {
6730 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6731 AssertRCReturn(rc, rc);
6732
6733 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6734 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6735 }
6736 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6737 NOREF(idCpu);
6738
6739 return VINF_SUCCESS;
6740}
6741
6742
6743/**
6744 * Leaves the VT-x session.
6745 *
6746 * @returns VBox status code.
6747 * @param pVM Pointer to the VM.
6748 * @param pVCpu Pointer to the VMCPU.
6749 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6750 * out-of-sync. Make sure to update the required fields
6751 * before using them.
6752 *
6753 * @remarks No-long-jmp zone!!!
6754 */
6755DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6756{
6757 HM_DISABLE_PREEMPT_IF_NEEDED();
6758 HMVMX_ASSERT_CPU_SAFE();
6759 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6760 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6761
6762 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6763 and done this from the VMXR0ThreadCtxCallback(). */
6764 if (!pVCpu->hm.s.fLeaveDone)
6765 {
6766 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6767 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6768 pVCpu->hm.s.fLeaveDone = true;
6769 }
6770
6771 /*
6772 * !!! IMPORTANT !!!
6773 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6774 */
6775
6776 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6777 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6778 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6779 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6780 VMMR0ThreadCtxHooksDeregister(pVCpu);
6781
6782 /* Leave HM context. This takes care of local init (term). */
6783 int rc = HMR0LeaveCpu(pVCpu);
6784
6785 HM_RESTORE_PREEMPT_IF_NEEDED();
6786
6787 return rc;
6788}
6789
6790
6791/**
6792 * Does the necessary state syncing before doing a longjmp to ring-3.
6793 *
6794 * @returns VBox status code.
6795 * @param pVM Pointer to the VM.
6796 * @param pVCpu Pointer to the VMCPU.
6797 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6798 * out-of-sync. Make sure to update the required fields
6799 * before using them.
6800 *
6801 * @remarks No-long-jmp zone!!!
6802 */
6803DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6804{
6805 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6806}
6807
6808
6809/**
6810 * Take necessary actions before going back to ring-3.
6811 *
6812 * An action requires us to go back to ring-3. This function does the necessary
6813 * steps before we can safely return to ring-3. This is not the same as longjmps
6814 * to ring-3, this is voluntary and prepares the guest so it may continue
6815 * executing outside HM (recompiler/IEM).
6816 *
6817 * @returns VBox status code.
6818 * @param pVM Pointer to the VM.
6819 * @param pVCpu Pointer to the VMCPU.
6820 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6821 * out-of-sync. Make sure to update the required fields
6822 * before using them.
6823 * @param rcExit The reason for exiting to ring-3. Can be
6824 * VINF_VMM_UNKNOWN_RING3_CALL.
6825 */
6826static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6827{
6828 Assert(pVM);
6829 Assert(pVCpu);
6830 Assert(pMixedCtx);
6831 HMVMX_ASSERT_PREEMPT_SAFE();
6832
6833 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6834 {
6835 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6836 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6837 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6838 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6839 }
6840
6841 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6842 VMMRZCallRing3Disable(pVCpu);
6843 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6844
6845 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6846 if (pVCpu->hm.s.Event.fPending)
6847 {
6848 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6849 Assert(!pVCpu->hm.s.Event.fPending);
6850 }
6851
6852 /* Save guest state and restore host state bits. */
6853 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6854 AssertRCReturn(rc, rc);
6855 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6856
6857 /* Sync recompiler state. */
6858 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6859 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6860 | CPUM_CHANGED_LDTR
6861 | CPUM_CHANGED_GDTR
6862 | CPUM_CHANGED_IDTR
6863 | CPUM_CHANGED_TR
6864 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6865 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6866 if ( pVM->hm.s.fNestedPaging
6867 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6868 {
6869 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6870 }
6871
6872 Assert(!pVCpu->hm.s.fClearTrapFlag);
6873
6874 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6875 if (rcExit != VINF_EM_RAW_INTERRUPT)
6876 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6877
6878 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6879
6880 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6881 VMMRZCallRing3RemoveNotification(pVCpu);
6882 VMMRZCallRing3Enable(pVCpu);
6883
6884 return rc;
6885}
6886
6887
6888/**
6889 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6890 * longjump to ring-3 and possibly get preempted.
6891 *
6892 * @returns VBox status code.
6893 * @param pVCpu Pointer to the VMCPU.
6894 * @param enmOperation The operation causing the ring-3 longjump.
6895 * @param pvUser Opaque pointer to the guest-CPU context. The data
6896 * may be out-of-sync. Make sure to update the required
6897 * fields before using them.
6898 */
6899DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6900{
6901 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6902 {
6903 /*
6904 * !!! IMPORTANT !!!
6905 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6906 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6907 */
6908 VMMRZCallRing3RemoveNotification(pVCpu);
6909 VMMRZCallRing3Disable(pVCpu);
6910 HM_DISABLE_PREEMPT_IF_NEEDED();
6911
6912 PVM pVM = pVCpu->CTX_SUFF(pVM);
6913 if (CPUMIsGuestFPUStateActive(pVCpu))
6914 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6915
6916 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6917
6918#if HC_ARCH_BITS == 64
6919 /* Restore host-state bits that VT-x only restores partially. */
6920 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6921 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6922 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6923 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6924
6925 /* Restore the host MSRs as we're leaving VT-x context. */
6926 if ( pVM->hm.s.fAllow64BitGuests
6927 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6928 {
6929 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6930 }
6931#endif
6932 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6933 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6934 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6935 {
6936 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6937 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6938 }
6939
6940 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6941 VMMR0ThreadCtxHooksDeregister(pVCpu);
6942
6943 HMR0LeaveCpu(pVCpu);
6944 HM_RESTORE_PREEMPT_IF_NEEDED();
6945 return VINF_SUCCESS;
6946 }
6947
6948 Assert(pVCpu);
6949 Assert(pvUser);
6950 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6951 HMVMX_ASSERT_PREEMPT_SAFE();
6952
6953 VMMRZCallRing3Disable(pVCpu);
6954 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6955
6956 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6957 enmOperation));
6958
6959 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6960 AssertRCReturn(rc, rc);
6961
6962 VMMRZCallRing3Enable(pVCpu);
6963 return VINF_SUCCESS;
6964}
6965
6966
6967/**
6968 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6969 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6970 *
6971 * @param pVCpu Pointer to the VMCPU.
6972 */
6973DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6974{
6975 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6976 {
6977 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6978 {
6979 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6980 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6981 AssertRC(rc);
6982 Log4(("Setup interrupt-window exiting\n"));
6983 }
6984 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6985}
6986
6987
6988/**
6989 * Clears the interrupt-window exiting control in the VMCS.
6990 *
6991 * @param pVCpu Pointer to the VMCPU.
6992 */
6993DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
6994{
6995 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
6996 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6997 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6998 AssertRC(rc);
6999 Log4(("Cleared interrupt-window exiting\n"));
7000}
7001
7002
7003/**
7004 * Evaluates the event to be delivered to the guest and sets it as the pending
7005 * event.
7006 *
7007 * @param pVCpu Pointer to the VMCPU.
7008 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7009 * out-of-sync. Make sure to update the required fields
7010 * before using them.
7011 */
7012static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7013{
7014 Assert(!pVCpu->hm.s.Event.fPending);
7015
7016 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7017 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7018 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7019 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7020
7021 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7022 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7023 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7024 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7025 Assert(!TRPMHasTrap(pVCpu));
7026
7027 /** @todo SMI. SMIs take priority over NMIs. */
7028 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7029 {
7030 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7031 if ( !fBlockMovSS
7032 && !fBlockSti)
7033 {
7034 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7035 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7036 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7037 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7038
7039 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7040 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7041 }
7042 else
7043 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7044 }
7045 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7046 && !pVCpu->hm.s.fSingleInstruction)
7047 {
7048 /*
7049 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7050 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
7051 * evaluated here and not set as pending, solely based on the force-flags.
7052 */
7053 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7054 AssertRC(rc);
7055 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7056 if ( !fBlockInt
7057 && !fBlockSti
7058 && !fBlockMovSS)
7059 {
7060 uint8_t u8Interrupt;
7061 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7062 if (RT_SUCCESS(rc))
7063 {
7064 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7065 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7066 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7067
7068 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7069 }
7070 else
7071 {
7072 /** @todo Does this actually happen? If not turn it into an assertion. */
7073 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7074 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7075 }
7076 }
7077 else
7078 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7079 }
7080}
7081
7082
7083/**
7084 * Sets a pending-debug exception to be delivered to the guest if the guest is
7085 * single-stepping.
7086 *
7087 * @param pVCpu Pointer to the VMCPU.
7088 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7089 * out-of-sync. Make sure to update the required fields
7090 * before using them.
7091 */
7092DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7093{
7094 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7095 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7096 {
7097 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7098 AssertRC(rc);
7099 }
7100}
7101
7102
7103/**
7104 * Injects any pending events into the guest if the guest is in a state to
7105 * receive them.
7106 *
7107 * @returns VBox status code (informational status codes included).
7108 * @param pVCpu Pointer to the VMCPU.
7109 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7110 * out-of-sync. Make sure to update the required fields
7111 * before using them.
7112 */
7113static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7114{
7115 HMVMX_ASSERT_PREEMPT_SAFE();
7116 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7117
7118 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7119 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7120 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7121 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7122
7123 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7124 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7125 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7126 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7127 Assert(!TRPMHasTrap(pVCpu));
7128
7129 int rc = VINF_SUCCESS;
7130 if (pVCpu->hm.s.Event.fPending)
7131 {
7132 /*
7133 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7134 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7135 * ended up enabling interrupts outside VT-x.
7136 */
7137 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7138 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7139 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7140 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7141 {
7142 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7143 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7144 }
7145#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
7146 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7147 {
7148 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7149 AssertRCReturn(rc, rc);
7150 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7151 Assert(!fBlockInt);
7152 Assert(!fBlockSti);
7153 Assert(!fBlockMovSS);
7154 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7155 }
7156 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7157 {
7158 Assert(!fBlockSti);
7159 Assert(!fBlockMovSS);
7160 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7161 }
7162#endif
7163 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7164 (uint8_t)uIntType));
7165 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7166 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7167 AssertRCReturn(rc, rc);
7168
7169 /* Update the interruptibility-state as it could have been changed by
7170 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7171 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7172 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7173
7174#ifdef VBOX_WITH_STATISTICS
7175 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7176 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7177 else
7178 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7179#endif
7180 }
7181
7182 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7183 if ( fBlockSti
7184 || fBlockMovSS)
7185 {
7186 if ( !pVCpu->hm.s.fSingleInstruction
7187 && !DBGFIsStepping(pVCpu))
7188 {
7189 /*
7190 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7191 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7192 * See Intel spec. 27.3.4 "Saving Non-Register State".
7193 */
7194 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7195 AssertRCReturn(rc2, rc2);
7196 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7197 }
7198 else if (pMixedCtx->eflags.Bits.u1TF)
7199 {
7200 /*
7201 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7202 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7203 */
7204 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7205 uIntrState = 0;
7206 }
7207 }
7208
7209 /*
7210 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7211 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7212 */
7213 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7214 AssertRC(rc2);
7215
7216 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7217 NOREF(fBlockMovSS); NOREF(fBlockSti);
7218 return rc;
7219}
7220
7221
7222/**
7223 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7224 *
7225 * @param pVCpu Pointer to the VMCPU.
7226 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7227 * out-of-sync. Make sure to update the required fields
7228 * before using them.
7229 */
7230DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7231{
7232 NOREF(pMixedCtx);
7233 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7234 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7235}
7236
7237
7238/**
7239 * Injects a double-fault (#DF) exception into the VM.
7240 *
7241 * @returns VBox status code (informational status code included).
7242 * @param pVCpu Pointer to the VMCPU.
7243 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7244 * out-of-sync. Make sure to update the required fields
7245 * before using them.
7246 */
7247DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7248{
7249 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7250 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7251 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7252 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7253 puIntrState);
7254}
7255
7256
7257/**
7258 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7259 *
7260 * @param pVCpu Pointer to the VMCPU.
7261 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7262 * out-of-sync. Make sure to update the required fields
7263 * before using them.
7264 */
7265DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7266{
7267 NOREF(pMixedCtx);
7268 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7269 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7270 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7271}
7272
7273
7274/**
7275 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7276 *
7277 * @param pVCpu Pointer to the VMCPU.
7278 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7279 * out-of-sync. Make sure to update the required fields
7280 * before using them.
7281 * @param cbInstr The value of RIP that is to be pushed on the guest
7282 * stack.
7283 */
7284DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7285{
7286 NOREF(pMixedCtx);
7287 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7288 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7289 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7290}
7291
7292
7293/**
7294 * Injects a general-protection (#GP) fault into the VM.
7295 *
7296 * @returns VBox status code (informational status code included).
7297 * @param pVCpu Pointer to the VMCPU.
7298 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7299 * out-of-sync. Make sure to update the required fields
7300 * before using them.
7301 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7302 * mode, i.e. in real-mode it's not valid).
7303 * @param u32ErrorCode The error code associated with the #GP.
7304 */
7305DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7306 uint32_t *puIntrState)
7307{
7308 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7309 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7310 if (fErrorCodeValid)
7311 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7312 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7313 puIntrState);
7314}
7315
7316
7317/**
7318 * Sets a general-protection (#GP) exception as pending-for-injection into the
7319 * VM.
7320 *
7321 * @param pVCpu Pointer to the VMCPU.
7322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7323 * out-of-sync. Make sure to update the required fields
7324 * before using them.
7325 * @param u32ErrorCode The error code associated with the #GP.
7326 */
7327DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7328{
7329 NOREF(pMixedCtx);
7330 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7331 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7332 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7333 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7334}
7335
7336
7337/**
7338 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7339 *
7340 * @param pVCpu Pointer to the VMCPU.
7341 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7342 * out-of-sync. Make sure to update the required fields
7343 * before using them.
7344 * @param uVector The software interrupt vector number.
7345 * @param cbInstr The value of RIP that is to be pushed on the guest
7346 * stack.
7347 */
7348DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7349{
7350 NOREF(pMixedCtx);
7351 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7352 if ( uVector == X86_XCPT_BP
7353 || uVector == X86_XCPT_OF)
7354 {
7355 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7356 }
7357 else
7358 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7359 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7360}
7361
7362
7363/**
7364 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7365 * stack.
7366 *
7367 * @returns VBox status code (information status code included).
7368 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7369 * @param pVM Pointer to the VM.
7370 * @param pMixedCtx Pointer to the guest-CPU context.
7371 * @param uValue The value to push to the guest stack.
7372 */
7373DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7374{
7375 /*
7376 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7377 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7378 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7379 */
7380 if (pMixedCtx->sp == 1)
7381 return VINF_EM_RESET;
7382 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7383 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7384 AssertRCReturn(rc, rc);
7385 return rc;
7386}
7387
7388
7389/**
7390 * Injects an event into the guest upon VM-entry by updating the relevant fields
7391 * in the VM-entry area in the VMCS.
7392 *
7393 * @returns VBox status code (informational error codes included).
7394 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7395 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7396 *
7397 * @param pVCpu Pointer to the VMCPU.
7398 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7399 * be out-of-sync. Make sure to update the required
7400 * fields before using them.
7401 * @param u64IntInfo The VM-entry interruption-information field.
7402 * @param cbInstr The VM-entry instruction length in bytes (for
7403 * software interrupts, exceptions and privileged
7404 * software exceptions).
7405 * @param u32ErrCode The VM-entry exception error code.
7406 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7407 * @param puIntrState Pointer to the current guest interruptibility-state.
7408 * This interruptibility-state will be updated if
7409 * necessary. This cannot not be NULL.
7410 *
7411 * @remarks Requires CR0!
7412 * @remarks No-long-jump zone!!!
7413 */
7414static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7415 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7416{
7417 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7418 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7419 Assert(puIntrState);
7420 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7421
7422 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7423 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7424
7425#ifdef VBOX_STRICT
7426 /* Validate the error-code-valid bit for hardware exceptions. */
7427 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7428 {
7429 switch (uVector)
7430 {
7431 case X86_XCPT_PF:
7432 case X86_XCPT_DF:
7433 case X86_XCPT_TS:
7434 case X86_XCPT_NP:
7435 case X86_XCPT_SS:
7436 case X86_XCPT_GP:
7437 case X86_XCPT_AC:
7438 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7439 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7440 /* fallthru */
7441 default:
7442 break;
7443 }
7444 }
7445#endif
7446
7447 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7448 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7449 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7450
7451 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7452
7453 /* We require CR0 to check if the guest is in real-mode. */
7454 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7455 AssertRCReturn(rc, rc);
7456
7457 /*
7458 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7459 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7460 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7461 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7462 */
7463 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7464 {
7465 PVM pVM = pVCpu->CTX_SUFF(pVM);
7466 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7467 {
7468 Assert(PDMVmmDevHeapIsEnabled(pVM));
7469 Assert(pVM->hm.s.vmx.pRealModeTSS);
7470
7471 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7472 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7473 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7474 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7475 AssertRCReturn(rc, rc);
7476 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7477
7478 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7479 const size_t cbIdtEntry = sizeof(X86IDTR16);
7480 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7481 {
7482 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7483 if (uVector == X86_XCPT_DF)
7484 return VINF_EM_RESET;
7485 else if (uVector == X86_XCPT_GP)
7486 {
7487 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7488 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7489 }
7490
7491 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7492 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7493 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7494 }
7495
7496 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7497 uint16_t uGuestIp = pMixedCtx->ip;
7498 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7499 {
7500 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7501 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7502 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7503 }
7504 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7505 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7506
7507 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7508 X86IDTR16 IdtEntry;
7509 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7510 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7511 AssertRCReturn(rc, rc);
7512
7513 /* Construct the stack frame for the interrupt/exception handler. */
7514 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7515 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7516 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7517 AssertRCReturn(rc, rc);
7518
7519 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7520 if (rc == VINF_SUCCESS)
7521 {
7522 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7523 pMixedCtx->rip = IdtEntry.offSel;
7524 pMixedCtx->cs.Sel = IdtEntry.uSel;
7525 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7526 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7527 && uVector == X86_XCPT_PF)
7528 {
7529 pMixedCtx->cr2 = GCPtrFaultAddress;
7530 }
7531
7532 /* If any other guest-state bits are changed here, make sure to update
7533 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7534 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7535 | HM_CHANGED_GUEST_RIP
7536 | HM_CHANGED_GUEST_RFLAGS
7537 | HM_CHANGED_GUEST_RSP);
7538
7539 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7540 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7541 {
7542 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7543 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7544 Log4(("Clearing inhibition due to STI.\n"));
7545 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7546 }
7547 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7548
7549 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7550 it, if we are returning to ring-3 before executing guest code. */
7551 pVCpu->hm.s.Event.fPending = false;
7552 }
7553 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7554 return rc;
7555 }
7556 else
7557 {
7558 /*
7559 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7560 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7561 */
7562 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7563 }
7564 }
7565
7566 /* Validate. */
7567 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7568 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7569 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7570
7571 /* Inject. */
7572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7573 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7574 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7575 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7576
7577 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7578 && uVector == X86_XCPT_PF)
7579 {
7580 pMixedCtx->cr2 = GCPtrFaultAddress;
7581 }
7582
7583 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7584 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7585
7586 AssertRCReturn(rc, rc);
7587 return rc;
7588}
7589
7590
7591/**
7592 * Clears the interrupt-window exiting control in the VMCS and if necessary
7593 * clears the current event in the VMCS as well.
7594 *
7595 * @returns VBox status code.
7596 * @param pVCpu Pointer to the VMCPU.
7597 *
7598 * @remarks Use this function only to clear events that have not yet been
7599 * delivered to the guest but are injected in the VMCS!
7600 * @remarks No-long-jump zone!!!
7601 */
7602static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7603{
7604 int rc;
7605 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7606
7607 /* Clear interrupt-window exiting control. */
7608 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7609 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7610
7611 if (!pVCpu->hm.s.Event.fPending)
7612 return;
7613
7614#ifdef VBOX_STRICT
7615 uint32_t u32EntryInfo;
7616 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7617 AssertRC(rc);
7618 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7619#endif
7620
7621 /* Clear the entry-interruption field (including the valid bit). */
7622 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7623 AssertRC(rc);
7624
7625 /* Clear the pending debug exception field. */
7626 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7627 AssertRC(rc);
7628}
7629
7630
7631/**
7632 * Enters the VT-x session.
7633 *
7634 * @returns VBox status code.
7635 * @param pVM Pointer to the VM.
7636 * @param pVCpu Pointer to the VMCPU.
7637 * @param pCpu Pointer to the CPU info struct.
7638 */
7639VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7640{
7641 AssertPtr(pVM);
7642 AssertPtr(pVCpu);
7643 Assert(pVM->hm.s.vmx.fSupported);
7644 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7645 NOREF(pCpu); NOREF(pVM);
7646
7647 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7648 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7649
7650#ifdef VBOX_STRICT
7651 /* Make sure we're in VMX root mode. */
7652 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7653 if (!(u32HostCR4 & X86_CR4_VMXE))
7654 {
7655 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7656 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7657 }
7658#endif
7659
7660 /*
7661 * Load the VCPU's VMCS as the current (and active) one.
7662 */
7663 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7664 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7665 if (RT_FAILURE(rc))
7666 return rc;
7667
7668 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7669 pVCpu->hm.s.fLeaveDone = false;
7670 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7671
7672 return VINF_SUCCESS;
7673}
7674
7675
7676/**
7677 * The thread-context callback (only on platforms which support it).
7678 *
7679 * @param enmEvent The thread-context event.
7680 * @param pVCpu Pointer to the VMCPU.
7681 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7682 * @thread EMT(pVCpu)
7683 */
7684VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7685{
7686 NOREF(fGlobalInit);
7687
7688 switch (enmEvent)
7689 {
7690 case RTTHREADCTXEVENT_PREEMPTING:
7691 {
7692 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7693 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7694 VMCPU_ASSERT_EMT(pVCpu);
7695
7696 PVM pVM = pVCpu->CTX_SUFF(pVM);
7697 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7698
7699 /* No longjmps (logger flushes, locks) in this fragile context. */
7700 VMMRZCallRing3Disable(pVCpu);
7701 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7702
7703 /*
7704 * Restore host-state (FPU, debug etc.)
7705 */
7706 if (!pVCpu->hm.s.fLeaveDone)
7707 {
7708 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7709 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7710 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7711 pVCpu->hm.s.fLeaveDone = true;
7712 }
7713
7714 /* Leave HM context, takes care of local init (term). */
7715 int rc = HMR0LeaveCpu(pVCpu);
7716 AssertRC(rc); NOREF(rc);
7717
7718 /* Restore longjmp state. */
7719 VMMRZCallRing3Enable(pVCpu);
7720 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7721 break;
7722 }
7723
7724 case RTTHREADCTXEVENT_RESUMED:
7725 {
7726 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7727 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7728 VMCPU_ASSERT_EMT(pVCpu);
7729
7730 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7731 VMMRZCallRing3Disable(pVCpu);
7732 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7733
7734 /* Initialize the bare minimum state required for HM. This takes care of
7735 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7736 int rc = HMR0EnterCpu(pVCpu);
7737 AssertRC(rc);
7738 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7739
7740 /* Load the active VMCS as the current one. */
7741 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7742 {
7743 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7744 AssertRC(rc); NOREF(rc);
7745 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7746 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7747 }
7748 pVCpu->hm.s.fLeaveDone = false;
7749
7750 /* Restore longjmp state. */
7751 VMMRZCallRing3Enable(pVCpu);
7752 break;
7753 }
7754
7755 default:
7756 break;
7757 }
7758}
7759
7760
7761/**
7762 * Saves the host state in the VMCS host-state.
7763 * Sets up the VM-exit MSR-load area.
7764 *
7765 * The CPU state will be loaded from these fields on every successful VM-exit.
7766 *
7767 * @returns VBox status code.
7768 * @param pVM Pointer to the VM.
7769 * @param pVCpu Pointer to the VMCPU.
7770 *
7771 * @remarks No-long-jump zone!!!
7772 */
7773static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7774{
7775 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7776
7777 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7778 return VINF_SUCCESS;
7779
7780 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7781 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7782
7783 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7784 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7785
7786 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7787 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7788
7789 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7790 return rc;
7791}
7792
7793
7794/**
7795 * Saves the host state in the VMCS host-state.
7796 *
7797 * @returns VBox status code.
7798 * @param pVM Pointer to the VM.
7799 * @param pVCpu Pointer to the VMCPU.
7800 *
7801 * @remarks No-long-jump zone!!!
7802 */
7803VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7804{
7805 AssertPtr(pVM);
7806 AssertPtr(pVCpu);
7807
7808 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7809
7810 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7811 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7812 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7813 return hmR0VmxSaveHostState(pVM, pVCpu);
7814}
7815
7816
7817/**
7818 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7819 * loaded from these fields on every successful VM-entry.
7820 *
7821 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7822 * Sets up the VM-entry controls.
7823 * Sets up the appropriate VMX non-root function to execute guest code based on
7824 * the guest CPU mode.
7825 *
7826 * @returns VBox status code.
7827 * @param pVM Pointer to the VM.
7828 * @param pVCpu Pointer to the VMCPU.
7829 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7830 * out-of-sync. Make sure to update the required fields
7831 * before using them.
7832 *
7833 * @remarks No-long-jump zone!!!
7834 */
7835static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7836{
7837 AssertPtr(pVM);
7838 AssertPtr(pVCpu);
7839 AssertPtr(pMixedCtx);
7840 HMVMX_ASSERT_PREEMPT_SAFE();
7841
7842#ifdef LOG_ENABLED
7843 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7844 * probably not initialized yet? Anyway this will do for now.
7845 *
7846 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7847 * interface and disable ring-3 calls when thread-context hooks are not
7848 * available. */
7849 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7850 VMMR0LogFlushDisable(pVCpu);
7851#endif
7852
7853 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7854
7855 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7856
7857 /* Determine real-on-v86 mode. */
7858 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7859 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7860 && CPUMIsGuestInRealModeEx(pMixedCtx))
7861 {
7862 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7863 }
7864
7865 /*
7866 * Load the guest-state into the VMCS.
7867 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7868 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7869 */
7870 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7871 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7872
7873 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7874 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7875 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7876
7877 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7878 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7879 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7880
7881 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7882 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7883
7884 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7885 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7886
7887 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7888 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7889 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7890
7891 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7892 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7893
7894 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7895 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7896
7897 /*
7898 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7899 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7900 */
7901 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7902 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7903
7904 /* Clear any unused and reserved bits. */
7905 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7906
7907#ifdef LOG_ENABLED
7908 /* Only reenable log-flushing if the caller has it enabled. */
7909 if (!fCallerDisabledLogFlush)
7910 VMMR0LogFlushEnable(pVCpu);
7911#endif
7912
7913 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7914 return rc;
7915}
7916
7917
7918/**
7919 * Loads the state shared between the host and guest into the VMCS.
7920 *
7921 * @param pVM Pointer to the VM.
7922 * @param pVCpu Pointer to the VMCPU.
7923 * @param pCtx Pointer to the guest-CPU context.
7924 *
7925 * @remarks No-long-jump zone!!!
7926 */
7927static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7928{
7929 NOREF(pVM);
7930
7931 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7932 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7933
7934 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7935 {
7936 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7937 AssertRC(rc);
7938 }
7939
7940 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7941 {
7942 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7943 AssertRC(rc);
7944
7945 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7946 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7947 {
7948 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7949 AssertRC(rc);
7950 }
7951 }
7952
7953 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7954 {
7955#if HC_ARCH_BITS == 64
7956 if (pVM->hm.s.fAllow64BitGuests)
7957 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7958#endif
7959 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7960 }
7961
7962 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7963 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7964}
7965
7966
7967/**
7968 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7969 *
7970 * @param pVM Pointer to the VM.
7971 * @param pVCpu Pointer to the VMCPU.
7972 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7973 * out-of-sync. Make sure to update the required fields
7974 * before using them.
7975 */
7976DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7977{
7978 HMVMX_ASSERT_PREEMPT_SAFE();
7979
7980 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7981#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7982 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7983#endif
7984
7985 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7986 {
7987 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7988 AssertRC(rc);
7989 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7990 }
7991 else if (HMCPU_CF_VALUE(pVCpu))
7992 {
7993 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7994 AssertRC(rc);
7995 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7996 }
7997
7998 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7999 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8000 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8001 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8002
8003#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8004 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8005 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8006 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8007#endif
8008}
8009
8010
8011/**
8012 * Does the preparations before executing guest code in VT-x.
8013 *
8014 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8015 * recompiler. We must be cautious what we do here regarding committing
8016 * guest-state information into the VMCS assuming we assuredly execute the
8017 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8018 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8019 * so that the recompiler can (and should) use them when it resumes guest
8020 * execution. Otherwise such operations must be done when we can no longer
8021 * exit to ring-3.
8022 *
8023 * @returns Strict VBox status code.
8024 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8025 * have been disabled.
8026 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8027 * double-fault into the guest.
8028 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8029 *
8030 * @param pVM Pointer to the VM.
8031 * @param pVCpu Pointer to the VMCPU.
8032 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8033 * out-of-sync. Make sure to update the required fields
8034 * before using them.
8035 * @param pVmxTransient Pointer to the VMX transient structure.
8036 */
8037static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8038{
8039 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8040
8041#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8042 PGMRZDynMapFlushAutoSet(pVCpu);
8043#endif
8044
8045 /* Check force flag actions that might require us to go back to ring-3. */
8046 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8047 if (rc != VINF_SUCCESS)
8048 return rc;
8049
8050#ifndef IEM_VERIFICATION_MODE_FULL
8051 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8052 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8053 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8054 {
8055 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8056 RTGCPHYS GCPhysApicBase;
8057 GCPhysApicBase = pMixedCtx->msrApicBase;
8058 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8059
8060 /* Unalias any existing mapping. */
8061 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8062 AssertRCReturn(rc, rc);
8063
8064 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8065 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8066 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8067 AssertRCReturn(rc, rc);
8068
8069 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8070 }
8071#endif /* !IEM_VERIFICATION_MODE_FULL */
8072
8073 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
8074 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8075
8076 /*
8077 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8078 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8079 */
8080 if (TRPMHasTrap(pVCpu))
8081 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8082 else if (!pVCpu->hm.s.Event.fPending)
8083 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8084
8085 /*
8086 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8087 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8088 */
8089 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8090 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8091 {
8092 Assert(rc == VINF_EM_RESET);
8093 return rc;
8094 }
8095
8096 /*
8097 * No longjmps to ring-3 from this point on!!!
8098 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8099 * This also disables flushing of the R0-logger instance (if any).
8100 */
8101 VMMRZCallRing3Disable(pVCpu);
8102
8103 /*
8104 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8105 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8106 *
8107 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8108 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8109 *
8110 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8111 * executing guest code.
8112 */
8113 pVmxTransient->uEflags = ASMIntDisableFlags();
8114 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8115 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8116 {
8117 hmR0VmxClearEventVmcs(pVCpu);
8118 ASMSetFlags(pVmxTransient->uEflags);
8119 VMMRZCallRing3Enable(pVCpu);
8120 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8121 return VINF_EM_RAW_TO_R3;
8122 }
8123 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8124 {
8125 hmR0VmxClearEventVmcs(pVCpu);
8126 ASMSetFlags(pVmxTransient->uEflags);
8127 VMMRZCallRing3Enable(pVCpu);
8128 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8129 return VINF_EM_RAW_INTERRUPT;
8130 }
8131
8132 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8133 pVCpu->hm.s.Event.fPending = false;
8134
8135 return VINF_SUCCESS;
8136}
8137
8138
8139/**
8140 * Prepares to run guest code in VT-x and we've committed to doing so. This
8141 * means there is no backing out to ring-3 or anywhere else at this
8142 * point.
8143 *
8144 * @param pVM Pointer to the VM.
8145 * @param pVCpu Pointer to the VMCPU.
8146 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8147 * out-of-sync. Make sure to update the required fields
8148 * before using them.
8149 * @param pVmxTransient Pointer to the VMX transient structure.
8150 *
8151 * @remarks Called with preemption disabled.
8152 * @remarks No-long-jump zone!!!
8153 */
8154static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8155{
8156 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8157 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8158 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8159
8160 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8161 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8162
8163 /*
8164 * If we are injecting events to a real-on-v86 mode guest, we may have to update
8165 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8166 * Reload only the necessary state, the assertion will catch if other parts of the code
8167 * change.
8168 */
8169 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8170 {
8171 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8172 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8173 }
8174
8175#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8176 if (!CPUMIsGuestFPUStateActive(pVCpu))
8177 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8178 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8179#endif
8180
8181 if ( pVCpu->hm.s.fUseGuestFpu
8182 && !CPUMIsGuestFPUStateActive(pVCpu))
8183 {
8184 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8185 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8186 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8187 }
8188
8189 /*
8190 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8191 */
8192 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8193 && pVCpu->hm.s.vmx.cMsrs > 0)
8194 {
8195 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8196 }
8197
8198 /*
8199 * Load the host state bits as we may've been preempted (only happens when
8200 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8201 */
8202 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8203 {
8204 /* This ASSUMES that pfnStartVM has been set up already. */
8205 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8206 AssertRC(rc);
8207 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8208 }
8209 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8210
8211 /*
8212 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8213 */
8214 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8215 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8216 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8217
8218 /* Store status of the shared guest-host state at the time of VM-entry. */
8219#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8220 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8221 {
8222 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8223 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8224 }
8225 else
8226#endif
8227 {
8228 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8229 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8230 }
8231 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8232
8233 /*
8234 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8235 */
8236 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8237 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8238
8239 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8240 RTCPUID idCurrentCpu = pCpu->idCpu;
8241 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8242 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8243 {
8244 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8245 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8246 }
8247
8248 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8249 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8250 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8251 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8252
8253 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8254
8255 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8256 to start executing. */
8257
8258 /*
8259 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8260 */
8261 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8262 {
8263 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8264 {
8265 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8266 AssertRC(rc2);
8267 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8268 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8269 true /* fUpdateHostMsr */);
8270 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8271 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8272 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8273 }
8274 else
8275 {
8276 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8277 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8278 }
8279 }
8280#ifdef VBOX_STRICT
8281 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8282#endif
8283}
8284
8285
8286/**
8287 * Performs some essential restoration of state after running guest code in
8288 * VT-x.
8289 *
8290 * @param pVM Pointer to the VM.
8291 * @param pVCpu Pointer to the VMCPU.
8292 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8293 * out-of-sync. Make sure to update the required fields
8294 * before using them.
8295 * @param pVmxTransient Pointer to the VMX transient structure.
8296 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8297 *
8298 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8299 *
8300 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8301 * unconditionally when it is safe to do so.
8302 */
8303static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8304{
8305 NOREF(pVM);
8306
8307 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8308
8309 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8310 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8311 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8312 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8313 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8314
8315 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8316 {
8317 /** @todo Find a way to fix hardcoding a guestimate. */
8318 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8319 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8320 }
8321
8322 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8323 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8324 Assert(!(ASMGetFlags() & X86_EFL_IF));
8325 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8326
8327#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8328 if (CPUMIsGuestFPUStateActive(pVCpu))
8329 {
8330 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8331 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8332 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8333 }
8334#endif
8335
8336#if HC_ARCH_BITS == 64
8337 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8338#endif
8339 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8340 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8341 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8342
8343 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8344 uint32_t uExitReason;
8345 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8346 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8347 AssertRC(rc);
8348 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8349 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8350
8351 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8352 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8353 {
8354 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8355 pVmxTransient->fVMEntryFailed));
8356 return;
8357 }
8358
8359 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8360 {
8361 /* Update the guest interruptibility-state from the VMCS. */
8362 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8363#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8364 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8365 AssertRC(rc);
8366#endif
8367 /*
8368 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8369 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8370 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8371 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8372 */
8373 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8374 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8375 {
8376 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8377 AssertRC(rc);
8378 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8379 }
8380 }
8381}
8382
8383
8384
8385/**
8386 * Runs the guest code using VT-x the normal way.
8387 *
8388 * @returns VBox status code.
8389 * @param pVM Pointer to the VM.
8390 * @param pVCpu Pointer to the VMCPU.
8391 * @param pCtx Pointer to the guest-CPU context.
8392 *
8393 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8394 */
8395static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8396{
8397 VMXTRANSIENT VmxTransient;
8398 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8399 int rc = VERR_INTERNAL_ERROR_5;
8400 uint32_t cLoops = 0;
8401
8402 for (;; cLoops++)
8403 {
8404 Assert(!HMR0SuspendPending());
8405 HMVMX_ASSERT_CPU_SAFE();
8406
8407 /* Preparatory work for running guest code, this may force us to return
8408 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8409 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8410 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8411 if (rc != VINF_SUCCESS)
8412 break;
8413
8414 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8415 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8416 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8417
8418 /* Restore any residual host-state and save any bits shared between host
8419 and guest into the guest-CPU state. Re-enables interrupts! */
8420 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8421
8422 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8423 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8424 {
8425 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8426 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8427 return rc;
8428 }
8429
8430 /* Handle the VM-exit. */
8431 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8432 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8433 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8434 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8435 HMVMX_START_EXIT_DISPATCH_PROF();
8436#ifdef HMVMX_USE_FUNCTION_TABLE
8437 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8438#else
8439 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8440#endif
8441 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8442 if (rc != VINF_SUCCESS)
8443 break;
8444 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8445 {
8446 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8447 rc = VINF_EM_RAW_INTERRUPT;
8448 break;
8449 }
8450 }
8451
8452 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8453 return rc;
8454}
8455
8456
8457/**
8458 * Single steps guest code using VT-x.
8459 *
8460 * @returns VBox status code.
8461 * @param pVM Pointer to the VM.
8462 * @param pVCpu Pointer to the VMCPU.
8463 * @param pCtx Pointer to the guest-CPU context.
8464 *
8465 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8466 */
8467static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8468{
8469 VMXTRANSIENT VmxTransient;
8470 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8471 int rc = VERR_INTERNAL_ERROR_5;
8472 uint32_t cLoops = 0;
8473 uint16_t uCsStart = pCtx->cs.Sel;
8474 uint64_t uRipStart = pCtx->rip;
8475
8476 for (;; cLoops++)
8477 {
8478 Assert(!HMR0SuspendPending());
8479 HMVMX_ASSERT_CPU_SAFE();
8480
8481 /* Preparatory work for running guest code, this may force us to return
8482 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8483 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8484 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8485 if (rc != VINF_SUCCESS)
8486 break;
8487
8488 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8489 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8490 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8491
8492 /* Restore any residual host-state and save any bits shared between host
8493 and guest into the guest-CPU state. Re-enables interrupts! */
8494 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8495
8496 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8497 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8498 {
8499 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8500 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8501 return rc;
8502 }
8503
8504 /* Handle the VM-exit. */
8505 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8507 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8508 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8509 HMVMX_START_EXIT_DISPATCH_PROF();
8510#ifdef HMVMX_USE_FUNCTION_TABLE
8511 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8512#else
8513 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8514#endif
8515 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8516 if (rc != VINF_SUCCESS)
8517 break;
8518 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8519 {
8520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8521 rc = VINF_EM_RAW_INTERRUPT;
8522 break;
8523 }
8524
8525 /*
8526 * Did the RIP change, if so, consider it a single step.
8527 * Otherwise, make sure one of the TFs gets set.
8528 */
8529 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8530 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8531 AssertRCReturn(rc2, rc2);
8532 if ( pCtx->rip != uRipStart
8533 || pCtx->cs.Sel != uCsStart)
8534 {
8535 rc = VINF_EM_DBG_STEPPED;
8536 break;
8537 }
8538 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8539 }
8540
8541 /*
8542 * Clear the X86_EFL_TF if necessary.
8543 */
8544 if (pVCpu->hm.s.fClearTrapFlag)
8545 {
8546 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8547 AssertRCReturn(rc2, rc2);
8548 pVCpu->hm.s.fClearTrapFlag = false;
8549 pCtx->eflags.Bits.u1TF = 0;
8550 }
8551 /** @todo there seems to be issues with the resume flag when the monitor trap
8552 * flag is pending without being used. Seen early in bios init when
8553 * accessing APIC page in prot mode. */
8554
8555 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8556 return rc;
8557}
8558
8559
8560/**
8561 * Runs the guest code using VT-x.
8562 *
8563 * @returns VBox status code.
8564 * @param pVM Pointer to the VM.
8565 * @param pVCpu Pointer to the VMCPU.
8566 * @param pCtx Pointer to the guest-CPU context.
8567 */
8568VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8569{
8570 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8571 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8572 HMVMX_ASSERT_PREEMPT_SAFE();
8573
8574 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8575
8576 int rc;
8577 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8578 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8579 else
8580 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8581
8582 if (rc == VERR_EM_INTERPRETER)
8583 rc = VINF_EM_RAW_EMULATE_INSTR;
8584 else if (rc == VINF_EM_RESET)
8585 rc = VINF_EM_TRIPLE_FAULT;
8586
8587 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8588 if (RT_FAILURE(rc2))
8589 {
8590 pVCpu->hm.s.u32HMError = rc;
8591 rc = rc2;
8592 }
8593 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8594 return rc;
8595}
8596
8597
8598#ifndef HMVMX_USE_FUNCTION_TABLE
8599DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8600{
8601#ifdef DEBUG_ramshankar
8602# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8603# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8604#endif
8605 int rc;
8606 switch (rcReason)
8607 {
8608 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8609 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8610 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8611 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8612 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8613 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8614 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8615 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8616 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8617 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8618 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8619 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8620 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8621 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8622 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8623 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8624 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8625 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8626 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8627 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8628 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8629 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8630 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8631 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8632 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8633 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8634 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8635 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8636 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8637 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8638 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8639 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8640 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8641
8642 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8643 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8644 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8645 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8646 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8647 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8648 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8649 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8650 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8651
8652 case VMX_EXIT_VMCALL:
8653 case VMX_EXIT_VMCLEAR:
8654 case VMX_EXIT_VMLAUNCH:
8655 case VMX_EXIT_VMPTRLD:
8656 case VMX_EXIT_VMPTRST:
8657 case VMX_EXIT_VMREAD:
8658 case VMX_EXIT_VMRESUME:
8659 case VMX_EXIT_VMWRITE:
8660 case VMX_EXIT_VMXOFF:
8661 case VMX_EXIT_VMXON:
8662 case VMX_EXIT_INVEPT:
8663 case VMX_EXIT_INVVPID:
8664 case VMX_EXIT_VMFUNC:
8665 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8666 break;
8667 default:
8668 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8669 break;
8670 }
8671 return rc;
8672}
8673#endif
8674
8675#ifdef DEBUG
8676/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8677# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8678 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8679
8680# define HMVMX_ASSERT_PREEMPT_CPUID() \
8681 do \
8682 { \
8683 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8684 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8685 } while (0)
8686
8687# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8688 do { \
8689 AssertPtr(pVCpu); \
8690 AssertPtr(pMixedCtx); \
8691 AssertPtr(pVmxTransient); \
8692 Assert(pVmxTransient->fVMEntryFailed == false); \
8693 Assert(ASMIntAreEnabled()); \
8694 HMVMX_ASSERT_PREEMPT_SAFE(); \
8695 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8696 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8697 HMVMX_ASSERT_PREEMPT_SAFE(); \
8698 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8699 HMVMX_ASSERT_PREEMPT_CPUID(); \
8700 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8701 } while (0)
8702
8703# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8704 do { \
8705 Log4Func(("\n")); \
8706 } while (0)
8707#else /* Release builds */
8708# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8709 do { \
8710 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8711 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8712 } while (0)
8713# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8714#endif
8715
8716
8717/**
8718 * Advances the guest RIP after reading it from the VMCS.
8719 *
8720 * @returns VBox status code.
8721 * @param pVCpu Pointer to the VMCPU.
8722 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8723 * out-of-sync. Make sure to update the required fields
8724 * before using them.
8725 * @param pVmxTransient Pointer to the VMX transient structure.
8726 *
8727 * @remarks No-long-jump zone!!!
8728 */
8729DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8730{
8731 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8732 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8733 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8734 AssertRCReturn(rc, rc);
8735
8736 pMixedCtx->rip += pVmxTransient->cbInstr;
8737 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8738
8739 /*
8740 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8741 * pending debug exception field as it takes care of priority of events.
8742 *
8743 * See Intel spec. 32.2.1 "Debug Exceptions".
8744 */
8745 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8746
8747 return rc;
8748}
8749
8750
8751/**
8752 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8753 * and update error record fields accordingly.
8754 *
8755 * @return VMX_IGS_* return codes.
8756 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8757 * wrong with the guest state.
8758 *
8759 * @param pVM Pointer to the VM.
8760 * @param pVCpu Pointer to the VMCPU.
8761 * @param pCtx Pointer to the guest-CPU state.
8762 */
8763static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8764{
8765#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8766#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8767 uError = (err); \
8768 break; \
8769 } else do { } while (0)
8770
8771 int rc;
8772 uint32_t uError = VMX_IGS_ERROR;
8773 uint32_t u32Val;
8774 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8775
8776 do
8777 {
8778 /*
8779 * CR0.
8780 */
8781 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8782 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8783 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8784 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8785 if (fUnrestrictedGuest)
8786 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8787
8788 uint32_t u32GuestCR0;
8789 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8790 AssertRCBreak(rc);
8791 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8792 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8793 if ( !fUnrestrictedGuest
8794 && (u32GuestCR0 & X86_CR0_PG)
8795 && !(u32GuestCR0 & X86_CR0_PE))
8796 {
8797 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8798 }
8799
8800 /*
8801 * CR4.
8802 */
8803 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8804 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8805
8806 uint32_t u32GuestCR4;
8807 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8808 AssertRCBreak(rc);
8809 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8810 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8811
8812 /*
8813 * IA32_DEBUGCTL MSR.
8814 */
8815 uint64_t u64Val;
8816 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8817 AssertRCBreak(rc);
8818 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8819 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8820 {
8821 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8822 }
8823 uint64_t u64DebugCtlMsr = u64Val;
8824
8825#ifdef VBOX_STRICT
8826 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8827 AssertRCBreak(rc);
8828 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8829#endif
8830 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8831
8832 /*
8833 * RIP and RFLAGS.
8834 */
8835 uint32_t u32Eflags;
8836#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8837 if (HMVMX_IS_64BIT_HOST_MODE())
8838 {
8839 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8840 AssertRCBreak(rc);
8841 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8842 if ( !fLongModeGuest
8843 || !pCtx->cs.Attr.n.u1Long)
8844 {
8845 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8846 }
8847 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8848 * must be identical if the "IA32e mode guest" VM-entry control is 1
8849 * and CS.L is 1. No check applies if the CPU supports 64
8850 * linear-address bits. */
8851
8852 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8853 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8854 AssertRCBreak(rc);
8855 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8856 VMX_IGS_RFLAGS_RESERVED);
8857 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8858 u32Eflags = u64Val;
8859 }
8860 else
8861#endif
8862 {
8863 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8864 AssertRCBreak(rc);
8865 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8866 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8867 }
8868
8869 if ( fLongModeGuest
8870 || ( fUnrestrictedGuest
8871 && !(u32GuestCR0 & X86_CR0_PE)))
8872 {
8873 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8874 }
8875
8876 uint32_t u32EntryInfo;
8877 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8878 AssertRCBreak(rc);
8879 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8880 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8881 {
8882 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8883 }
8884
8885 /*
8886 * 64-bit checks.
8887 */
8888#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8889 if (HMVMX_IS_64BIT_HOST_MODE())
8890 {
8891 if ( fLongModeGuest
8892 && !fUnrestrictedGuest)
8893 {
8894 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8895 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8896 }
8897
8898 if ( !fLongModeGuest
8899 && (u32GuestCR4 & X86_CR4_PCIDE))
8900 {
8901 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8902 }
8903
8904 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8905 * 51:32 beyond the processor's physical-address width are 0. */
8906
8907 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8908 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8909 {
8910 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8911 }
8912
8913 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8914 AssertRCBreak(rc);
8915 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8916
8917 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8918 AssertRCBreak(rc);
8919 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8920 }
8921#endif
8922
8923 /*
8924 * PERF_GLOBAL MSR.
8925 */
8926 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8927 {
8928 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8929 AssertRCBreak(rc);
8930 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8931 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8932 }
8933
8934 /*
8935 * PAT MSR.
8936 */
8937 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8938 {
8939 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8940 AssertRCBreak(rc);
8941 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8942 for (unsigned i = 0; i < 8; i++)
8943 {
8944 uint8_t u8Val = (u64Val & 0x7);
8945 if ( u8Val != 0 /* UC */
8946 || u8Val != 1 /* WC */
8947 || u8Val != 4 /* WT */
8948 || u8Val != 5 /* WP */
8949 || u8Val != 6 /* WB */
8950 || u8Val != 7 /* UC- */)
8951 {
8952 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8953 }
8954 u64Val >>= 3;
8955 }
8956 }
8957
8958 /*
8959 * EFER MSR.
8960 */
8961 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8962 {
8963 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8964 AssertRCBreak(rc);
8965 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8966 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8967 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8968 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8969 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8970 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8971 }
8972
8973 /*
8974 * Segment registers.
8975 */
8976 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8977 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8978 if (!(u32Eflags & X86_EFL_VM))
8979 {
8980 /* CS */
8981 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8982 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8983 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8984 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8985 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8986 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8987 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8988 /* CS cannot be loaded with NULL in protected mode. */
8989 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8990 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8991 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8992 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8993 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8994 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8995 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8996 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8997 else
8998 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8999
9000 /* SS */
9001 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9002 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9003 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9004 if ( !(pCtx->cr0 & X86_CR0_PE)
9005 || pCtx->cs.Attr.n.u4Type == 3)
9006 {
9007 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9008 }
9009 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9010 {
9011 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9012 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9013 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9014 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9015 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9016 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9017 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9018 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9019 }
9020
9021 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9022 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9023 {
9024 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9025 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9026 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9027 || pCtx->ds.Attr.n.u4Type > 11
9028 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9029 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9030 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9031 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9032 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9033 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9034 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9035 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9036 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9037 }
9038 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9039 {
9040 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9041 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9042 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9043 || pCtx->es.Attr.n.u4Type > 11
9044 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9045 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9046 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9047 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9048 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9049 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9050 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9051 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9052 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9053 }
9054 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9055 {
9056 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9057 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9058 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9059 || pCtx->fs.Attr.n.u4Type > 11
9060 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9061 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9062 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9063 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9064 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9065 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9066 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9067 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9068 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9069 }
9070 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9071 {
9072 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9073 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9074 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9075 || pCtx->gs.Attr.n.u4Type > 11
9076 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9077 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9078 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9079 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9080 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9081 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9082 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9083 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9084 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9085 }
9086 /* 64-bit capable CPUs. */
9087#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9088 if (HMVMX_IS_64BIT_HOST_MODE())
9089 {
9090 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9091 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9092 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9093 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9094 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9095 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9096 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9097 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9098 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9099 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9100 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9101 }
9102#endif
9103 }
9104 else
9105 {
9106 /* V86 mode checks. */
9107 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9108 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9109 {
9110 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9111 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9112 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9113 }
9114 else
9115 {
9116 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9117 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9118 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9119 }
9120
9121 /* CS */
9122 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9123 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9124 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9125 /* SS */
9126 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9127 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9128 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9129 /* DS */
9130 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9131 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9132 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9133 /* ES */
9134 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9135 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9136 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9137 /* FS */
9138 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9139 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9140 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9141 /* GS */
9142 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9143 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9144 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9145 /* 64-bit capable CPUs. */
9146#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9147 if (HMVMX_IS_64BIT_HOST_MODE())
9148 {
9149 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9150 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9151 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9152 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9153 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9154 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9155 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9156 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9157 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9158 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9159 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9160 }
9161#endif
9162 }
9163
9164 /*
9165 * TR.
9166 */
9167 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9168 /* 64-bit capable CPUs. */
9169#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9170 if (HMVMX_IS_64BIT_HOST_MODE())
9171 {
9172 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9173 }
9174#endif
9175 if (fLongModeGuest)
9176 {
9177 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9178 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9179 }
9180 else
9181 {
9182 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9183 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9184 VMX_IGS_TR_ATTR_TYPE_INVALID);
9185 }
9186 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9187 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9188 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9189 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9190 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9191 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9192 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9193 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9194
9195 /*
9196 * GDTR and IDTR.
9197 */
9198#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9199 if (HMVMX_IS_64BIT_HOST_MODE())
9200 {
9201 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9202 AssertRCBreak(rc);
9203 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9204
9205 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9206 AssertRCBreak(rc);
9207 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9208 }
9209#endif
9210
9211 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9212 AssertRCBreak(rc);
9213 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9214
9215 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9216 AssertRCBreak(rc);
9217 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9218
9219 /*
9220 * Guest Non-Register State.
9221 */
9222 /* Activity State. */
9223 uint32_t u32ActivityState;
9224 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9225 AssertRCBreak(rc);
9226 HMVMX_CHECK_BREAK( !u32ActivityState
9227 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9228 VMX_IGS_ACTIVITY_STATE_INVALID);
9229 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9230 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9231 uint32_t u32IntrState;
9232 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9233 AssertRCBreak(rc);
9234 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9235 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9236 {
9237 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9238 }
9239
9240 /** @todo Activity state and injecting interrupts. Left as a todo since we
9241 * currently don't use activity states but ACTIVE. */
9242
9243 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9244 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9245
9246 /* Guest interruptibility-state. */
9247 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9248 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9249 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9250 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9251 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9252 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9253 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9254 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9255 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9256 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9257 {
9258 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9259 {
9260 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9261 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9262 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9263 }
9264 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9265 {
9266 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9267 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9268 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9269 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9270 }
9271 }
9272 /** @todo Assumes the processor is not in SMM. */
9273 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9274 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9275 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9276 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9277 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9278 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9279 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9280 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9281 {
9282 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9283 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9284 }
9285
9286 /* Pending debug exceptions. */
9287 if (HMVMX_IS_64BIT_HOST_MODE())
9288 {
9289 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9290 AssertRCBreak(rc);
9291 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9292 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9293 u32Val = u64Val; /* For pending debug exceptions checks below. */
9294 }
9295 else
9296 {
9297 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9298 AssertRCBreak(rc);
9299 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9300 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9301 }
9302
9303 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9304 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9305 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9306 {
9307 if ( (u32Eflags & X86_EFL_TF)
9308 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9309 {
9310 /* Bit 14 is PendingDebug.BS. */
9311 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9312 }
9313 if ( !(u32Eflags & X86_EFL_TF)
9314 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9315 {
9316 /* Bit 14 is PendingDebug.BS. */
9317 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9318 }
9319 }
9320
9321 /* VMCS link pointer. */
9322 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9323 AssertRCBreak(rc);
9324 if (u64Val != UINT64_C(0xffffffffffffffff))
9325 {
9326 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9327 /** @todo Bits beyond the processor's physical-address width MBZ. */
9328 /** @todo 32-bit located in memory referenced by value of this field (as a
9329 * physical address) must contain the processor's VMCS revision ID. */
9330 /** @todo SMM checks. */
9331 }
9332
9333 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9334 * not using Nested Paging? */
9335 if ( pVM->hm.s.fNestedPaging
9336 && !fLongModeGuest
9337 && CPUMIsGuestInPAEModeEx(pCtx))
9338 {
9339 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9340 AssertRCBreak(rc);
9341 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9342
9343 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9344 AssertRCBreak(rc);
9345 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9346
9347 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9348 AssertRCBreak(rc);
9349 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9350
9351 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9352 AssertRCBreak(rc);
9353 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9354 }
9355
9356 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9357 if (uError == VMX_IGS_ERROR)
9358 uError = VMX_IGS_REASON_NOT_FOUND;
9359 } while (0);
9360
9361 pVCpu->hm.s.u32HMError = uError;
9362 return uError;
9363
9364#undef HMVMX_ERROR_BREAK
9365#undef HMVMX_CHECK_BREAK
9366}
9367
9368/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9369/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9370/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9371
9372/** @name VM-exit handlers.
9373 * @{
9374 */
9375
9376/**
9377 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9378 */
9379HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9380{
9381 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9382 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9383 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9384 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9385 return VINF_SUCCESS;
9386 return VINF_EM_RAW_INTERRUPT;
9387}
9388
9389
9390/**
9391 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9392 */
9393HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9394{
9395 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9396 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9397
9398 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9399 AssertRCReturn(rc, rc);
9400
9401 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9402 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9403 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9404 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9405
9406 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9407 {
9408 /*
9409 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9410 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9411 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9412 *
9413 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9414 */
9415 VMXDispatchHostNmi();
9416 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9417 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9418 return VINF_SUCCESS;
9419 }
9420
9421 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9422 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9423 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9424 {
9425 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9426 return VINF_SUCCESS;
9427 }
9428 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9429 {
9430 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9431 return rc;
9432 }
9433
9434 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9435 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9436 switch (uIntType)
9437 {
9438 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9439 Assert(uVector == X86_XCPT_DB);
9440 /* no break */
9441 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9442 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9443 /* no break */
9444 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9445 {
9446 switch (uVector)
9447 {
9448 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9449 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9450 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9451 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9452 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9453 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9454#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9455 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9456 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9457 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9458 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9459 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9460 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9461 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9462 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9463 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9464 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9465#endif
9466 default:
9467 {
9468 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9469 AssertRCReturn(rc, rc);
9470
9471 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9472 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9473 {
9474 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9475 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9476 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9477
9478 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9479 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9480 AssertRCReturn(rc, rc);
9481 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9482 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9483 0 /* GCPtrFaultAddress */);
9484 AssertRCReturn(rc, rc);
9485 }
9486 else
9487 {
9488 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9489 pVCpu->hm.s.u32HMError = uVector;
9490 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9491 }
9492 break;
9493 }
9494 }
9495 break;
9496 }
9497
9498 default:
9499 {
9500 pVCpu->hm.s.u32HMError = uExitIntInfo;
9501 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9502 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9503 break;
9504 }
9505 }
9506 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9507 return rc;
9508}
9509
9510
9511/**
9512 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9513 */
9514HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9515{
9516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9517
9518 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9519 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9520
9521 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9522 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9523 return VINF_SUCCESS;
9524}
9525
9526
9527/**
9528 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9529 */
9530HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9531{
9532 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9533 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9534 HMVMX_RETURN_UNEXPECTED_EXIT();
9535}
9536
9537
9538/**
9539 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9540 */
9541HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9542{
9543 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9544 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9545 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9546}
9547
9548
9549/**
9550 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9551 */
9552HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9553{
9554 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9555 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9556 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9557}
9558
9559
9560/**
9561 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9562 */
9563HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9564{
9565 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9566 PVM pVM = pVCpu->CTX_SUFF(pVM);
9567 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9568 if (RT_LIKELY(rc == VINF_SUCCESS))
9569 {
9570 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9571 Assert(pVmxTransient->cbInstr == 2);
9572 }
9573 else
9574 {
9575 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9576 rc = VERR_EM_INTERPRETER;
9577 }
9578 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9579 return rc;
9580}
9581
9582
9583/**
9584 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9585 */
9586HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9587{
9588 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9589 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9590 AssertRCReturn(rc, rc);
9591
9592 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9593 return VINF_EM_RAW_EMULATE_INSTR;
9594
9595 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9596 HMVMX_RETURN_UNEXPECTED_EXIT();
9597}
9598
9599
9600/**
9601 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9602 */
9603HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9604{
9605 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9606 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9607 AssertRCReturn(rc, rc);
9608
9609 PVM pVM = pVCpu->CTX_SUFF(pVM);
9610 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9611 if (RT_LIKELY(rc == VINF_SUCCESS))
9612 {
9613 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9614 Assert(pVmxTransient->cbInstr == 2);
9615 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9616 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9617 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9618 }
9619 else
9620 {
9621 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9622 rc = VERR_EM_INTERPRETER;
9623 }
9624 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9625 return rc;
9626}
9627
9628
9629/**
9630 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9631 */
9632HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9633{
9634 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9635 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9636 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9637 AssertRCReturn(rc, rc);
9638
9639 PVM pVM = pVCpu->CTX_SUFF(pVM);
9640 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9641 if (RT_LIKELY(rc == VINF_SUCCESS))
9642 {
9643 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9644 Assert(pVmxTransient->cbInstr == 3);
9645 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9646 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9647 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9648 }
9649 else
9650 {
9651 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9652 rc = VERR_EM_INTERPRETER;
9653 }
9654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9655 return rc;
9656}
9657
9658
9659/**
9660 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9661 */
9662HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9663{
9664 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9665 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9666 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9667 AssertRCReturn(rc, rc);
9668
9669 PVM pVM = pVCpu->CTX_SUFF(pVM);
9670 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9671 if (RT_LIKELY(rc == VINF_SUCCESS))
9672 {
9673 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9674 Assert(pVmxTransient->cbInstr == 2);
9675 }
9676 else
9677 {
9678 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9679 rc = VERR_EM_INTERPRETER;
9680 }
9681 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9682 return rc;
9683}
9684
9685
9686/**
9687 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9688 */
9689HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9690{
9691 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9692 PVM pVM = pVCpu->CTX_SUFF(pVM);
9693 Assert(!pVM->hm.s.fNestedPaging);
9694
9695 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9696 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9697 AssertRCReturn(rc, rc);
9698
9699 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9700 rc = VBOXSTRICTRC_VAL(rc2);
9701 if (RT_LIKELY(rc == VINF_SUCCESS))
9702 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9703 else
9704 {
9705 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9706 pVmxTransient->uExitQualification, rc));
9707 }
9708 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9709 return rc;
9710}
9711
9712
9713/**
9714 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9715 */
9716HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9717{
9718 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9719 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9720 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9721 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9722 AssertRCReturn(rc, rc);
9723
9724 PVM pVM = pVCpu->CTX_SUFF(pVM);
9725 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9726 if (RT_LIKELY(rc == VINF_SUCCESS))
9727 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9728 else
9729 {
9730 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9731 rc = VERR_EM_INTERPRETER;
9732 }
9733 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9734 return rc;
9735}
9736
9737
9738/**
9739 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9740 */
9741HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9742{
9743 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9744 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9745 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9746 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9747 AssertRCReturn(rc, rc);
9748
9749 PVM pVM = pVCpu->CTX_SUFF(pVM);
9750 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9751 rc = VBOXSTRICTRC_VAL(rc2);
9752 if (RT_LIKELY( rc == VINF_SUCCESS
9753 || rc == VINF_EM_HALT))
9754 {
9755 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9756 AssertRCReturn(rc3, rc3);
9757
9758 if ( rc == VINF_EM_HALT
9759 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9760 {
9761 rc = VINF_SUCCESS;
9762 }
9763 }
9764 else
9765 {
9766 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9767 rc = VERR_EM_INTERPRETER;
9768 }
9769 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9770 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9771 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9772 return rc;
9773}
9774
9775
9776/**
9777 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9778 */
9779HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9780{
9781 /*
9782 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9783 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9784 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9785 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9786 */
9787 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9788 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9789 HMVMX_RETURN_UNEXPECTED_EXIT();
9790}
9791
9792
9793/**
9794 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9795 */
9796HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9797{
9798 /*
9799 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9800 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9801 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9802 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9803 */
9804 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9805 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9806 HMVMX_RETURN_UNEXPECTED_EXIT();
9807}
9808
9809
9810/**
9811 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9812 */
9813HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9814{
9815 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9817 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9818 HMVMX_RETURN_UNEXPECTED_EXIT();
9819}
9820
9821
9822/**
9823 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9824 */
9825HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9826{
9827 /*
9828 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9829 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9830 * See Intel spec. 25.3 "Other Causes of VM-exits".
9831 */
9832 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9833 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9834 HMVMX_RETURN_UNEXPECTED_EXIT();
9835}
9836
9837
9838/**
9839 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9840 * VM-exit.
9841 */
9842HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9843{
9844 /*
9845 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9846 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9847 *
9848 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9849 * See Intel spec. "23.8 Restrictions on VMX operation".
9850 */
9851 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9852 return VINF_SUCCESS;
9853}
9854
9855
9856/**
9857 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9858 * VM-exit.
9859 */
9860HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9861{
9862 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9863 return VINF_EM_RESET;
9864}
9865
9866
9867/**
9868 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9869 */
9870HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9871{
9872 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9873 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9874 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9875 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9876 AssertRCReturn(rc, rc);
9877
9878 pMixedCtx->rip++;
9879 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9880 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9881 rc = VINF_SUCCESS;
9882 else
9883 rc = VINF_EM_HALT;
9884
9885 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9886 return rc;
9887}
9888
9889
9890/**
9891 * VM-exit handler for instructions that result in a #UD exception delivered to
9892 * the guest.
9893 */
9894HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9895{
9896 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9897 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9898 return VINF_SUCCESS;
9899}
9900
9901
9902/**
9903 * VM-exit handler for expiry of the VMX preemption timer.
9904 */
9905HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9906{
9907 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9908
9909 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9910 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9911
9912 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9913 PVM pVM = pVCpu->CTX_SUFF(pVM);
9914 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9915 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9916 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9917}
9918
9919
9920/**
9921 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9922 */
9923HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9924{
9925 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9926
9927 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9928 /** @todo check if XSETBV is supported by the recompiler. */
9929 return VERR_EM_INTERPRETER;
9930}
9931
9932
9933/**
9934 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9935 */
9936HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9937{
9938 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9939
9940 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9941 /** @todo implement EMInterpretInvpcid() */
9942 return VERR_EM_INTERPRETER;
9943}
9944
9945
9946/**
9947 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9948 * Error VM-exit.
9949 */
9950HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9951{
9952 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9953 AssertRCReturn(rc, rc);
9954
9955 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9956 NOREF(uInvalidReason);
9957
9958#ifdef VBOX_STRICT
9959 uint32_t uIntrState;
9960 HMVMXHCUINTREG uHCReg;
9961 uint64_t u64Val;
9962 uint32_t u32Val;
9963
9964 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9965 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9966 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9967 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9968 AssertRCReturn(rc, rc);
9969
9970 Log4(("uInvalidReason %u\n", uInvalidReason));
9971 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9972 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9973 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9974 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9975
9976 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9977 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9978 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9979 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9980 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9981 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9982 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9983 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9984 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9985 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9986 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9987 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9988#else
9989 NOREF(pVmxTransient);
9990#endif
9991
9992 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9993 return VERR_VMX_INVALID_GUEST_STATE;
9994}
9995
9996
9997/**
9998 * VM-exit handler for VM-entry failure due to an MSR-load
9999 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10000 */
10001HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10002{
10003 NOREF(pVmxTransient);
10004 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10005 HMVMX_RETURN_UNEXPECTED_EXIT();
10006}
10007
10008
10009/**
10010 * VM-exit handler for VM-entry failure due to a machine-check event
10011 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10012 */
10013HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10014{
10015 NOREF(pVmxTransient);
10016 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10017 HMVMX_RETURN_UNEXPECTED_EXIT();
10018}
10019
10020
10021/**
10022 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10023 * theory.
10024 */
10025HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10026{
10027 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10028 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10029 return VERR_VMX_UNDEFINED_EXIT_CODE;
10030}
10031
10032
10033/**
10034 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10035 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10036 * Conditional VM-exit.
10037 */
10038HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10039{
10040 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10041
10042 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10043 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10044 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10045 return VERR_EM_INTERPRETER;
10046 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10047 HMVMX_RETURN_UNEXPECTED_EXIT();
10048}
10049
10050
10051/**
10052 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10053 */
10054HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10055{
10056 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10057
10058 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10059 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10060 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10061 return VERR_EM_INTERPRETER;
10062 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10063 HMVMX_RETURN_UNEXPECTED_EXIT();
10064}
10065
10066
10067/**
10068 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10069 */
10070HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10071{
10072 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10073
10074 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10075 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10076 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10077 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10078 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10079 {
10080 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10081 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10082 }
10083 AssertRCReturn(rc, rc);
10084 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10085
10086#ifdef VBOX_STRICT
10087 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10088 {
10089 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10090 {
10091 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10092 HMVMX_RETURN_UNEXPECTED_EXIT();
10093 }
10094# if HC_ARCH_BITS == 64
10095 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10096 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10097 {
10098 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10099 HMVMX_RETURN_UNEXPECTED_EXIT();
10100 }
10101# endif
10102 }
10103#endif
10104
10105 PVM pVM = pVCpu->CTX_SUFF(pVM);
10106 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10107 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10108 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10110
10111 if (RT_LIKELY(rc == VINF_SUCCESS))
10112 {
10113 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10114 Assert(pVmxTransient->cbInstr == 2);
10115 }
10116 return rc;
10117}
10118
10119
10120/**
10121 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10122 */
10123HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10124{
10125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10126 PVM pVM = pVCpu->CTX_SUFF(pVM);
10127 int rc = VINF_SUCCESS;
10128
10129 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10130 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10131 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10132 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10133 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10134 {
10135 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10136 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10137 }
10138 AssertRCReturn(rc, rc);
10139 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10140
10141 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10142 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10143 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10144
10145 if (RT_LIKELY(rc == VINF_SUCCESS))
10146 {
10147 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10148
10149 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10150 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10151 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10152 {
10153 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10154 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10155 EMInterpretWrmsr() changes it. */
10156 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10157 }
10158 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10159 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10160
10161 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10162 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10163 {
10164 switch (pMixedCtx->ecx)
10165 {
10166 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10167 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10168 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10169 case MSR_K8_FS_BASE: /* no break */
10170 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10171 default:
10172 {
10173 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10174 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10175#if HC_ARCH_BITS == 64
10176 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10177 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10178#endif
10179 break;
10180 }
10181 }
10182 }
10183#ifdef VBOX_STRICT
10184 else
10185 {
10186 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10187 switch (pMixedCtx->ecx)
10188 {
10189 case MSR_IA32_SYSENTER_CS:
10190 case MSR_IA32_SYSENTER_EIP:
10191 case MSR_IA32_SYSENTER_ESP:
10192 case MSR_K8_FS_BASE:
10193 case MSR_K8_GS_BASE:
10194 {
10195 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10196 HMVMX_RETURN_UNEXPECTED_EXIT();
10197 }
10198
10199 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10200 default:
10201 {
10202 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10203 {
10204 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10205 pMixedCtx->ecx));
10206 HMVMX_RETURN_UNEXPECTED_EXIT();
10207 }
10208
10209#if HC_ARCH_BITS == 64
10210 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10211 {
10212 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10213 HMVMX_RETURN_UNEXPECTED_EXIT();
10214 }
10215#endif
10216 break;
10217 }
10218 }
10219 }
10220#endif /* VBOX_STRICT */
10221 }
10222 return rc;
10223}
10224
10225
10226/**
10227 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10228 */
10229HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10230{
10231 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10232
10233 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10235 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10236 return VERR_EM_INTERPRETER;
10237 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10238 HMVMX_RETURN_UNEXPECTED_EXIT();
10239}
10240
10241
10242/**
10243 * VM-exit handler for when the TPR value is lowered below the specified
10244 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10245 */
10246HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10247{
10248 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10249 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10250
10251 /*
10252 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10253 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10254 * resume guest execution.
10255 */
10256 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10257 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10258 return VINF_SUCCESS;
10259}
10260
10261
10262/**
10263 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10264 * VM-exit.
10265 *
10266 * @retval VINF_SUCCESS when guest execution can continue.
10267 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10268 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10269 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10270 * recompiler.
10271 */
10272HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10273{
10274 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10275 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10276 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10277 AssertRCReturn(rc, rc);
10278
10279 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10280 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10281 PVM pVM = pVCpu->CTX_SUFF(pVM);
10282 switch (uAccessType)
10283 {
10284 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10285 {
10286#if 0
10287 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10288 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10289#else
10290 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10291 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10292 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10293#endif
10294 AssertRCReturn(rc, rc);
10295
10296 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10297 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10298 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10299 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10300
10301 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10302 {
10303 case 0: /* CR0 */
10304 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10305 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10306 break;
10307 case 2: /* CR2 */
10308 /* Nothing to do here, CR2 it's not part of the VMCS. */
10309 break;
10310 case 3: /* CR3 */
10311 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10312 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10313 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10314 break;
10315 case 4: /* CR4 */
10316 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10317 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10318 break;
10319 case 8: /* CR8 */
10320 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10321 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10322 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10323 break;
10324 default:
10325 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10326 break;
10327 }
10328
10329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10330 break;
10331 }
10332
10333 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10334 {
10335 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10336 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10337 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10338 AssertRCReturn(rc, rc);
10339 Assert( !pVM->hm.s.fNestedPaging
10340 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10341 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10342
10343 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10344 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10345 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10346
10347 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10348 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10349 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10350 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10352 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10353 break;
10354 }
10355
10356 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10357 {
10358 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10359 AssertRCReturn(rc, rc);
10360 rc = EMInterpretCLTS(pVM, pVCpu);
10361 AssertRCReturn(rc, rc);
10362 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10363 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10364 Log4(("CRX CLTS write rc=%d\n", rc));
10365 break;
10366 }
10367
10368 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10369 {
10370 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10371 AssertRCReturn(rc, rc);
10372 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10373 if (RT_LIKELY(rc == VINF_SUCCESS))
10374 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10376 Log4(("CRX LMSW write rc=%d\n", rc));
10377 break;
10378 }
10379
10380 default:
10381 {
10382 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10383 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10384 }
10385 }
10386
10387 /* Validate possible error codes. */
10388 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10389 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10390 if (RT_SUCCESS(rc))
10391 {
10392 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10393 AssertRCReturn(rc2, rc2);
10394 }
10395
10396 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10397 return rc;
10398}
10399
10400
10401/**
10402 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10403 * VM-exit.
10404 */
10405HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10406{
10407 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10408 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10409
10410 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10411 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10412 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10413 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10414 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10415 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10416 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10417 AssertRCReturn(rc2, rc2);
10418
10419 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10420 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10421 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10422 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10423 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10424 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10425 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10426 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10427
10428 /* I/O operation lookup arrays. */
10429 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10430 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10431
10432 VBOXSTRICTRC rcStrict;
10433 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10434 const uint32_t cbInstr = pVmxTransient->cbInstr;
10435 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10436 PVM pVM = pVCpu->CTX_SUFF(pVM);
10437 if (fIOString)
10438 {
10439#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10440 /*
10441 * INS/OUTS - I/O String instruction.
10442 *
10443 * Use instruction-information if available, otherwise fall back on
10444 * interpreting the instruction.
10445 */
10446 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10447 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10448 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10449 {
10450 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10451 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10452 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10453 AssertRCReturn(rc2, rc2);
10454 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10455 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10456 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10457 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10458 if (fIOWrite)
10459 {
10460 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10461 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10462 }
10463 else
10464 {
10465 /*
10466 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10467 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10468 * See Intel Instruction spec. for "INS".
10469 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10470 */
10471 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10472 }
10473 }
10474 else
10475 {
10476 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10477 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10478 AssertRCReturn(rc2, rc2);
10479 rcStrict = IEMExecOne(pVCpu);
10480 }
10481 /** @todo IEM needs to be setting these flags somehow. */
10482 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10483 fUpdateRipAlready = true;
10484#else
10485 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10486 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10487 if (RT_SUCCESS(rcStrict))
10488 {
10489 if (fIOWrite)
10490 {
10491 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10492 (DISCPUMODE)pDis->uAddrMode, cbValue);
10493 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10494 }
10495 else
10496 {
10497 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10498 (DISCPUMODE)pDis->uAddrMode, cbValue);
10499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10500 }
10501 }
10502 else
10503 {
10504 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10505 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10506 }
10507#endif
10508 }
10509 else
10510 {
10511 /*
10512 * IN/OUT - I/O instruction.
10513 */
10514 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10515 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10516 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10517 if (fIOWrite)
10518 {
10519 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10520 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10521 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10522 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10523 }
10524 else
10525 {
10526 uint32_t u32Result = 0;
10527 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10528 if (IOM_SUCCESS(rcStrict))
10529 {
10530 /* Save result of I/O IN instr. in AL/AX/EAX. */
10531 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10532 }
10533 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10534 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10535 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10536 }
10537 }
10538
10539 if (IOM_SUCCESS(rcStrict))
10540 {
10541 if (!fUpdateRipAlready)
10542 {
10543 pMixedCtx->rip += cbInstr;
10544 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10545 }
10546
10547 /*
10548 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10549 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10550 */
10551 if (fIOString)
10552 {
10553 /** @todo Single-step for INS/OUTS with REP prefix? */
10554 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10555 }
10556 else if (fStepping)
10557 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10558
10559 /*
10560 * If any I/O breakpoints are armed, we need to check if one triggered
10561 * and take appropriate action.
10562 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10563 */
10564 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10565 AssertRCReturn(rc2, rc2);
10566
10567 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10568 * execution engines about whether hyper BPs and such are pending. */
10569 uint32_t const uDr7 = pMixedCtx->dr[7];
10570 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10571 && X86_DR7_ANY_RW_IO(uDr7)
10572 && (pMixedCtx->cr4 & X86_CR4_DE))
10573 || DBGFBpIsHwIoArmed(pVM)))
10574 {
10575 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10576
10577 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10578 VMMRZCallRing3Disable(pVCpu);
10579 HM_DISABLE_PREEMPT_IF_NEEDED();
10580
10581 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10582
10583 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10584 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10585 {
10586 /* Raise #DB. */
10587 if (fIsGuestDbgActive)
10588 ASMSetDR6(pMixedCtx->dr[6]);
10589 if (pMixedCtx->dr[7] != uDr7)
10590 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10591
10592 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10593 }
10594 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10595 else if ( rcStrict2 != VINF_SUCCESS
10596 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10597 rcStrict = rcStrict2;
10598
10599 HM_RESTORE_PREEMPT_IF_NEEDED();
10600 VMMRZCallRing3Enable(pVCpu);
10601 }
10602 }
10603
10604#ifdef DEBUG
10605 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10606 Assert(!fIOWrite);
10607 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10608 Assert(fIOWrite);
10609 else
10610 {
10611 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10612 * statuses, that the VMM device and some others may return. See
10613 * IOM_SUCCESS() for guidance. */
10614 AssertMsg( RT_FAILURE(rcStrict)
10615 || rcStrict == VINF_SUCCESS
10616 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10617 || rcStrict == VINF_EM_DBG_BREAKPOINT
10618 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10619 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10620 }
10621#endif
10622
10623 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10624 return VBOXSTRICTRC_TODO(rcStrict);
10625}
10626
10627
10628/**
10629 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10630 * VM-exit.
10631 */
10632HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10633{
10634 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10635
10636 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10637 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10638 AssertRCReturn(rc, rc);
10639 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10640 {
10641 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10642 AssertRCReturn(rc, rc);
10643 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10644 {
10645 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10646
10647 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10648 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10649 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10650 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10651 {
10652 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10653 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10654
10655 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10656 Assert(!pVCpu->hm.s.Event.fPending);
10657 pVCpu->hm.s.Event.fPending = true;
10658 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10659 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10660 AssertRCReturn(rc, rc);
10661 if (fErrorCodeValid)
10662 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10663 else
10664 pVCpu->hm.s.Event.u32ErrCode = 0;
10665 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10666 && uVector == X86_XCPT_PF)
10667 {
10668 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10669 }
10670
10671 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10672 }
10673 }
10674 }
10675
10676 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10677 * emulation. */
10678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10679 return VERR_EM_INTERPRETER;
10680}
10681
10682
10683/**
10684 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10685 */
10686HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10687{
10688 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10689 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10690 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10691 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10692 AssertRCReturn(rc, rc);
10693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10694 return VINF_EM_DBG_STEPPED;
10695}
10696
10697
10698/**
10699 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10700 */
10701HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10702{
10703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10704
10705 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10706 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10707 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10708 return VINF_SUCCESS;
10709 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10710 return rc;
10711
10712#if 0
10713 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10714 * just sync the whole thing. */
10715 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10716#else
10717 /* Aggressive state sync. for now. */
10718 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10719 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10720 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10721#endif
10722 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10723 AssertRCReturn(rc, rc);
10724
10725 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10726 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10727 switch (uAccessType)
10728 {
10729 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10730 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10731 {
10732 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10733 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10734 {
10735 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10736 }
10737
10738 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10739 GCPhys &= PAGE_BASE_GC_MASK;
10740 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10741 PVM pVM = pVCpu->CTX_SUFF(pVM);
10742 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10743 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10744
10745 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10746 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10747 CPUMCTX2CORE(pMixedCtx), GCPhys);
10748 rc = VBOXSTRICTRC_VAL(rc2);
10749 Log4(("ApicAccess rc=%d\n", rc));
10750 if ( rc == VINF_SUCCESS
10751 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10752 || rc == VERR_PAGE_NOT_PRESENT)
10753 {
10754 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10755 | HM_CHANGED_GUEST_RSP
10756 | HM_CHANGED_GUEST_RFLAGS
10757 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10758 rc = VINF_SUCCESS;
10759 }
10760 break;
10761 }
10762
10763 default:
10764 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10765 rc = VINF_EM_RAW_EMULATE_INSTR;
10766 break;
10767 }
10768
10769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10770 return rc;
10771}
10772
10773
10774/**
10775 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10776 * VM-exit.
10777 */
10778HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10779{
10780 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10781
10782 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10783 if (pVmxTransient->fWasGuestDebugStateActive)
10784 {
10785 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10786 HMVMX_RETURN_UNEXPECTED_EXIT();
10787 }
10788
10789 int rc = VERR_INTERNAL_ERROR_5;
10790 if ( !DBGFIsStepping(pVCpu)
10791 && !pVCpu->hm.s.fSingleInstruction
10792 && !pVmxTransient->fWasHyperDebugStateActive)
10793 {
10794 /* Don't intercept MOV DRx and #DB any more. */
10795 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10796 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10797 AssertRCReturn(rc, rc);
10798
10799 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10800 {
10801#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10802 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10803 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10804 AssertRCReturn(rc, rc);
10805#endif
10806 }
10807
10808 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10809 VMMRZCallRing3Disable(pVCpu);
10810 HM_DISABLE_PREEMPT_IF_NEEDED();
10811
10812 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10813 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10814 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10815
10816 HM_RESTORE_PREEMPT_IF_NEEDED();
10817 VMMRZCallRing3Enable(pVCpu);
10818
10819#ifdef VBOX_WITH_STATISTICS
10820 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10821 AssertRCReturn(rc, rc);
10822 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10823 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10824 else
10825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10826#endif
10827 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10828 return VINF_SUCCESS;
10829 }
10830
10831 /*
10832 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10833 * Update the segment registers and DR7 from the CPU.
10834 */
10835 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10836 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10837 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10838 AssertRCReturn(rc, rc);
10839 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10840
10841 PVM pVM = pVCpu->CTX_SUFF(pVM);
10842 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10843 {
10844 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10845 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10846 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10847 if (RT_SUCCESS(rc))
10848 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10850 }
10851 else
10852 {
10853 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10854 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10855 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10857 }
10858
10859 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10860 if (RT_SUCCESS(rc))
10861 {
10862 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10863 AssertRCReturn(rc2, rc2);
10864 }
10865 return rc;
10866}
10867
10868
10869/**
10870 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10871 * Conditional VM-exit.
10872 */
10873HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10874{
10875 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10876 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10877
10878 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10879 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10880 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10881 return VINF_SUCCESS;
10882 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10883 return rc;
10884
10885 RTGCPHYS GCPhys = 0;
10886 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10887
10888#if 0
10889 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10890#else
10891 /* Aggressive state sync. for now. */
10892 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10893 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10894 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10895#endif
10896 AssertRCReturn(rc, rc);
10897
10898 /*
10899 * If we succeed, resume guest execution.
10900 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10901 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10902 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10903 * weird case. See @bugref{6043}.
10904 */
10905 PVM pVM = pVCpu->CTX_SUFF(pVM);
10906 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10907 rc = VBOXSTRICTRC_VAL(rc2);
10908 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10909 if ( rc == VINF_SUCCESS
10910 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10911 || rc == VERR_PAGE_NOT_PRESENT)
10912 {
10913 /* Successfully handled MMIO operation. */
10914 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10915 | HM_CHANGED_GUEST_RSP
10916 | HM_CHANGED_GUEST_RFLAGS
10917 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10918 rc = VINF_SUCCESS;
10919 }
10920 return rc;
10921}
10922
10923
10924/**
10925 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10926 * VM-exit.
10927 */
10928HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10929{
10930 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10931 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10932
10933 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10934 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10935 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10936 return VINF_SUCCESS;
10937 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10938 return rc;
10939
10940 RTGCPHYS GCPhys = 0;
10941 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10942 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10943#if 0
10944 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10945#else
10946 /* Aggressive state sync. for now. */
10947 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10948 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10949 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10950#endif
10951 AssertRCReturn(rc, rc);
10952
10953 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10954 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10955
10956 RTGCUINT uErrorCode = 0;
10957 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10958 uErrorCode |= X86_TRAP_PF_ID;
10959 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10960 uErrorCode |= X86_TRAP_PF_RW;
10961 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10962 uErrorCode |= X86_TRAP_PF_P;
10963
10964 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10965
10966 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10967 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10968
10969 /* Handle the pagefault trap for the nested shadow table. */
10970 PVM pVM = pVCpu->CTX_SUFF(pVM);
10971 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10972 TRPMResetTrap(pVCpu);
10973
10974 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10975 if ( rc == VINF_SUCCESS
10976 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10977 || rc == VERR_PAGE_NOT_PRESENT)
10978 {
10979 /* Successfully synced our nested page tables. */
10980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10981 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10982 | HM_CHANGED_GUEST_RSP
10983 | HM_CHANGED_GUEST_RFLAGS);
10984 return VINF_SUCCESS;
10985 }
10986
10987 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10988 return rc;
10989}
10990
10991/** @} */
10992
10993/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10994/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10995/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10996
10997/** @name VM-exit exception handlers.
10998 * @{
10999 */
11000
11001/**
11002 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11003 */
11004static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11005{
11006 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11008
11009 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11010 AssertRCReturn(rc, rc);
11011
11012 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11013 {
11014 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11015 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11016
11017 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11018 * provides VM-exit instruction length. If this causes problem later,
11019 * disassemble the instruction like it's done on AMD-V. */
11020 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11021 AssertRCReturn(rc2, rc2);
11022 return rc;
11023 }
11024
11025 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11026 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11027 return rc;
11028}
11029
11030
11031/**
11032 * VM-exit exception handler for #BP (Breakpoint exception).
11033 */
11034static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11035{
11036 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11038
11039 /** @todo Try optimize this by not saving the entire guest state unless
11040 * really needed. */
11041 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11042 AssertRCReturn(rc, rc);
11043
11044 PVM pVM = pVCpu->CTX_SUFF(pVM);
11045 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11046 if (rc == VINF_EM_RAW_GUEST_TRAP)
11047 {
11048 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11049 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11050 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11051 AssertRCReturn(rc, rc);
11052
11053 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11054 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11055 }
11056
11057 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11058 return rc;
11059}
11060
11061
11062/**
11063 * VM-exit exception handler for #DB (Debug exception).
11064 */
11065static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11066{
11067 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11068 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11069 Log6(("XcptDB\n"));
11070
11071 /*
11072 * Get the DR6-like values from the exit qualification and pass it to DBGF
11073 * for processing.
11074 */
11075 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11076 AssertRCReturn(rc, rc);
11077
11078 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11079 uint64_t uDR6 = X86_DR6_INIT_VAL;
11080 uDR6 |= ( pVmxTransient->uExitQualification
11081 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11082
11083 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11084 if (rc == VINF_EM_RAW_GUEST_TRAP)
11085 {
11086 /*
11087 * The exception was for the guest. Update DR6, DR7.GD and
11088 * IA32_DEBUGCTL.LBR before forwarding it.
11089 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11090 */
11091 VMMRZCallRing3Disable(pVCpu);
11092 HM_DISABLE_PREEMPT_IF_NEEDED();
11093
11094 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11095 pMixedCtx->dr[6] |= uDR6;
11096 if (CPUMIsGuestDebugStateActive(pVCpu))
11097 ASMSetDR6(pMixedCtx->dr[6]);
11098
11099 HM_RESTORE_PREEMPT_IF_NEEDED();
11100 VMMRZCallRing3Enable(pVCpu);
11101
11102 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11103 AssertRCReturn(rc, rc);
11104
11105 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11106 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11107
11108 /* Paranoia. */
11109 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11110 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11111
11112 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11113 AssertRCReturn(rc, rc);
11114
11115 /*
11116 * Raise #DB in the guest.
11117 *
11118 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11119 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11120 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11121 *
11122 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11123 */
11124 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11125 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11126 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11127 AssertRCReturn(rc, rc);
11128 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11129 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11130 return VINF_SUCCESS;
11131 }
11132
11133 /*
11134 * Not a guest trap, must be a hypervisor related debug event then.
11135 * Update DR6 in case someone is interested in it.
11136 */
11137 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11138 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11139 CPUMSetHyperDR6(pVCpu, uDR6);
11140
11141 return rc;
11142}
11143
11144
11145/**
11146 * VM-exit exception handler for #NM (Device-not-available exception: floating
11147 * point exception).
11148 */
11149static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11150{
11151 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11152
11153 /* We require CR0 and EFER. EFER is always up-to-date. */
11154 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11155 AssertRCReturn(rc, rc);
11156
11157 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11158 VMMRZCallRing3Disable(pVCpu);
11159 HM_DISABLE_PREEMPT_IF_NEEDED();
11160
11161 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11162 if (pVmxTransient->fWasGuestFPUStateActive)
11163 {
11164 rc = VINF_EM_RAW_GUEST_TRAP;
11165 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11166 }
11167 else
11168 {
11169#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11170 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11171#endif
11172 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11173 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11174 }
11175
11176 HM_RESTORE_PREEMPT_IF_NEEDED();
11177 VMMRZCallRing3Enable(pVCpu);
11178
11179 if (rc == VINF_SUCCESS)
11180 {
11181 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11182 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11183 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11184 pVCpu->hm.s.fUseGuestFpu = true;
11185 }
11186 else
11187 {
11188 /* Forward #NM to the guest. */
11189 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11190 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11191 AssertRCReturn(rc, rc);
11192 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11193 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11194 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11195 }
11196
11197 return VINF_SUCCESS;
11198}
11199
11200
11201/**
11202 * VM-exit exception handler for #GP (General-protection exception).
11203 *
11204 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11205 */
11206static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11207{
11208 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11209 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11210
11211 int rc = VERR_INTERNAL_ERROR_5;
11212 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11213 {
11214#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11215 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11216 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11217 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11218 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11219 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11220 AssertRCReturn(rc, rc);
11221 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11222 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11223 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11224 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11225 return rc;
11226#else
11227 /* We don't intercept #GP. */
11228 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11229 NOREF(pVmxTransient);
11230 return VERR_VMX_UNEXPECTED_EXCEPTION;
11231#endif
11232 }
11233
11234 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11235 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11236
11237 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11238 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11239 AssertRCReturn(rc, rc);
11240
11241 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11242 uint32_t cbOp = 0;
11243 PVM pVM = pVCpu->CTX_SUFF(pVM);
11244 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11245 if (RT_SUCCESS(rc))
11246 {
11247 rc = VINF_SUCCESS;
11248 Assert(cbOp == pDis->cbInstr);
11249 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11250 switch (pDis->pCurInstr->uOpcode)
11251 {
11252 case OP_CLI:
11253 {
11254 pMixedCtx->eflags.Bits.u1IF = 0;
11255 pMixedCtx->rip += pDis->cbInstr;
11256 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11257 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11258 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11259 break;
11260 }
11261
11262 case OP_STI:
11263 {
11264 pMixedCtx->eflags.Bits.u1IF = 1;
11265 pMixedCtx->rip += pDis->cbInstr;
11266 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11267 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11268 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11269 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11270 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11271 break;
11272 }
11273
11274 case OP_HLT:
11275 {
11276 rc = VINF_EM_HALT;
11277 pMixedCtx->rip += pDis->cbInstr;
11278 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11279 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11280 break;
11281 }
11282
11283 case OP_POPF:
11284 {
11285 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11286 uint32_t cbParm;
11287 uint32_t uMask;
11288 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11289 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11290 {
11291 cbParm = 4;
11292 uMask = 0xffffffff;
11293 }
11294 else
11295 {
11296 cbParm = 2;
11297 uMask = 0xffff;
11298 }
11299
11300 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11301 RTGCPTR GCPtrStack = 0;
11302 X86EFLAGS Eflags;
11303 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11304 &GCPtrStack);
11305 if (RT_SUCCESS(rc))
11306 {
11307 Assert(sizeof(Eflags.u32) >= cbParm);
11308 Eflags.u32 = 0;
11309 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11310 }
11311 if (RT_FAILURE(rc))
11312 {
11313 rc = VERR_EM_INTERPRETER;
11314 break;
11315 }
11316 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11317 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11318 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11319 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11320 pMixedCtx->esp += cbParm;
11321 pMixedCtx->esp &= uMask;
11322 pMixedCtx->rip += pDis->cbInstr;
11323 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11324 | HM_CHANGED_GUEST_RSP
11325 | HM_CHANGED_GUEST_RFLAGS);
11326 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11327 if (fStepping)
11328 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11329
11330 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11331 break;
11332 }
11333
11334 case OP_PUSHF:
11335 {
11336 uint32_t cbParm;
11337 uint32_t uMask;
11338 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11339 {
11340 cbParm = 4;
11341 uMask = 0xffffffff;
11342 }
11343 else
11344 {
11345 cbParm = 2;
11346 uMask = 0xffff;
11347 }
11348
11349 /* Get the stack pointer & push the contents of eflags onto the stack. */
11350 RTGCPTR GCPtrStack = 0;
11351 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11352 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11353 if (RT_FAILURE(rc))
11354 {
11355 rc = VERR_EM_INTERPRETER;
11356 break;
11357 }
11358 X86EFLAGS Eflags = pMixedCtx->eflags;
11359 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11360 Eflags.Bits.u1RF = 0;
11361 Eflags.Bits.u1VM = 0;
11362
11363 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11364 if (RT_FAILURE(rc))
11365 {
11366 rc = VERR_EM_INTERPRETER;
11367 break;
11368 }
11369 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11370 pMixedCtx->esp -= cbParm;
11371 pMixedCtx->esp &= uMask;
11372 pMixedCtx->rip += pDis->cbInstr;
11373 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11374 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11376 break;
11377 }
11378
11379 case OP_IRET:
11380 {
11381 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11382 * instruction reference. */
11383 RTGCPTR GCPtrStack = 0;
11384 uint32_t uMask = 0xffff;
11385 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11386 uint16_t aIretFrame[3];
11387 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11388 {
11389 rc = VERR_EM_INTERPRETER;
11390 break;
11391 }
11392 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11393 &GCPtrStack);
11394 if (RT_SUCCESS(rc))
11395 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11396 if (RT_FAILURE(rc))
11397 {
11398 rc = VERR_EM_INTERPRETER;
11399 break;
11400 }
11401 pMixedCtx->eip = 0;
11402 pMixedCtx->ip = aIretFrame[0];
11403 pMixedCtx->cs.Sel = aIretFrame[1];
11404 pMixedCtx->cs.ValidSel = aIretFrame[1];
11405 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11406 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11407 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11408 pMixedCtx->sp += sizeof(aIretFrame);
11409 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11410 | HM_CHANGED_GUEST_SEGMENT_REGS
11411 | HM_CHANGED_GUEST_RSP
11412 | HM_CHANGED_GUEST_RFLAGS);
11413 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11414 if (fStepping)
11415 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11416 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11417 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11418 break;
11419 }
11420
11421 case OP_INT:
11422 {
11423 uint16_t uVector = pDis->Param1.uValue & 0xff;
11424 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11425 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11426 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11427 break;
11428 }
11429
11430 case OP_INTO:
11431 {
11432 if (pMixedCtx->eflags.Bits.u1OF)
11433 {
11434 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11435 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11437 }
11438 break;
11439 }
11440
11441 default:
11442 {
11443 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11444 EMCODETYPE_SUPERVISOR);
11445 rc = VBOXSTRICTRC_VAL(rc2);
11446 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11447 /** @todo We have to set pending-debug exceptions here when the guest is
11448 * single-stepping depending on the instruction that was interpreted. */
11449 Log4(("#GP rc=%Rrc\n", rc));
11450 break;
11451 }
11452 }
11453 }
11454 else
11455 rc = VERR_EM_INTERPRETER;
11456
11457 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11458 ("#GP Unexpected rc=%Rrc\n", rc));
11459 return rc;
11460}
11461
11462
11463#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11464/**
11465 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11466 * the exception reported in the VMX transient structure back into the VM.
11467 *
11468 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11469 * up-to-date.
11470 */
11471static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11472{
11473 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11474
11475 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11476 hmR0VmxCheckExitDueToEventDelivery(). */
11477 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11478 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11479 AssertRCReturn(rc, rc);
11480 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11481
11482 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11483 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11484 return VINF_SUCCESS;
11485}
11486#endif
11487
11488
11489/**
11490 * VM-exit exception handler for #PF (Page-fault exception).
11491 */
11492static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11493{
11494 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11495 PVM pVM = pVCpu->CTX_SUFF(pVM);
11496 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11497 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11498 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11499 AssertRCReturn(rc, rc);
11500
11501#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11502 if (pVM->hm.s.fNestedPaging)
11503 {
11504 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11505 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11506 {
11507 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11508 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11509 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11510 }
11511 else
11512 {
11513 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11514 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11515 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11516 }
11517 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11518 return rc;
11519 }
11520#else
11521 Assert(!pVM->hm.s.fNestedPaging);
11522 NOREF(pVM);
11523#endif
11524
11525 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11526 AssertRCReturn(rc, rc);
11527
11528 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11529 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11530
11531 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11532 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11533 (RTGCPTR)pVmxTransient->uExitQualification);
11534
11535 Log4(("#PF: rc=%Rrc\n", rc));
11536 if (rc == VINF_SUCCESS)
11537 {
11538 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11539 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11540 * memory? We don't update the whole state here... */
11541 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11542 | HM_CHANGED_GUEST_RSP
11543 | HM_CHANGED_GUEST_RFLAGS
11544 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11545 TRPMResetTrap(pVCpu);
11546 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11547 return rc;
11548 }
11549 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11550 {
11551 if (!pVmxTransient->fVectoringPF)
11552 {
11553 /* It's a guest page fault and needs to be reflected to the guest. */
11554 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11555 TRPMResetTrap(pVCpu);
11556 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11557 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11558 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11559 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11560 }
11561 else
11562 {
11563 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11564 TRPMResetTrap(pVCpu);
11565 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11566 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11567 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11568 }
11569
11570 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11571 return VINF_SUCCESS;
11572 }
11573
11574 TRPMResetTrap(pVCpu);
11575 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11576 return rc;
11577}
11578
11579/** @} */
11580
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette