VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 51230

Last change on this file since 51230 was 51230, checked in by vboxsync, 11 years ago

VMM/HMVMXR0,HMSVMR0: debug build fixes.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 490.6 KB
Line 
1/* $Id: HMVMXR0.cpp 51230 2014-05-12 05:32:21Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_SAVE_FULL_GUEST_STATE
41# define HMVMX_SYNC_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44# define HMVMX_ALWAYS_TRAP_PF
45# define HMVMX_ALWAYS_SWAP_FPU_STATE
46# define HMVMX_ALWAYS_FLUSH_TLB
47# define HMVMX_ALWAYS_SWAP_EFER
48#endif
49
50
51/*******************************************************************************
52* Defined Constants And Macros *
53*******************************************************************************/
54#if defined(RT_ARCH_AMD64)
55# define HMVMX_IS_64BIT_HOST_MODE() (true)
56typedef RTHCUINTREG HMVMXHCUINTREG;
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58extern "C" uint32_t g_fVMXIs64bitHost;
59# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
60typedef uint64_t HMVMXHCUINTREG;
61#else
62# define HMVMX_IS_64BIT_HOST_MODE() (false)
63typedef RTHCUINTREG HMVMXHCUINTREG;
64#endif
65
66/** Use the function table. */
67#define HMVMX_USE_FUNCTION_TABLE
68
69/** Determine which tagged-TLB flush handler to use. */
70#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
71#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
72#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
73#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
74
75/** @name Updated-guest-state flags.
76 * @{ */
77#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
78#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
79#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
80#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
81#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
82#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
83#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
84#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
85#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
86#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
87#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
88#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
89#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
90#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
91#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
92#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
93#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
94#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
95#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
96#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
97 | HMVMX_UPDATED_GUEST_RSP \
98 | HMVMX_UPDATED_GUEST_RFLAGS \
99 | HMVMX_UPDATED_GUEST_CR0 \
100 | HMVMX_UPDATED_GUEST_CR3 \
101 | HMVMX_UPDATED_GUEST_CR4 \
102 | HMVMX_UPDATED_GUEST_GDTR \
103 | HMVMX_UPDATED_GUEST_IDTR \
104 | HMVMX_UPDATED_GUEST_LDTR \
105 | HMVMX_UPDATED_GUEST_TR \
106 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
107 | HMVMX_UPDATED_GUEST_DEBUG \
108 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
113 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
114 | HMVMX_UPDATED_GUEST_APIC_STATE)
115/** @} */
116
117/** @name
118 * Flags to skip redundant reads of some common VMCS fields that are not part of
119 * the guest-CPU state but are in the transient structure.
120 */
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
122#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
128/** @} */
129
130/** @name
131 * States of the VMCS.
132 *
133 * This does not reflect all possible VMCS states but currently only those
134 * needed for maintaining the VMCS consistently even when thread-context hooks
135 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
136 */
137#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
138#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
139#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
140/** @} */
141
142/**
143 * Exception bitmap mask for real-mode guests (real-on-v86).
144 *
145 * We need to intercept all exceptions manually (except #PF). #NM is also
146 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
147 * even in real-mode if we have Nested Paging support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*******************************************************************************
198* Structures and Typedefs *
199*******************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG uEflags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u6Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringPF;
279} VMXTRANSIENT;
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285/** Pointer to VMX transient state. */
286typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289/**
290 * MSR-bitmap read permissions.
291 */
292typedef enum VMXMSREXITREAD
293{
294 /** Reading this MSR causes a VM-exit. */
295 VMXMSREXIT_INTERCEPT_READ = 0xb,
296 /** Reading this MSR does not cause a VM-exit. */
297 VMXMSREXIT_PASSTHRU_READ
298} VMXMSREXITREAD;
299/** Pointer to MSR-bitmap read permissions. */
300typedef VMXMSREXITREAD* PVMXMSREXITREAD;
301
302/**
303 * MSR-bitmap write permissions.
304 */
305typedef enum VMXMSREXITWRITE
306{
307 /** Writing to this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
309 /** Writing to this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_WRITE
311} VMXMSREXITWRITE;
312/** Pointer to MSR-bitmap write permissions. */
313typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
314
315
316/**
317 * VMX VM-exit handler.
318 *
319 * @returns VBox status code.
320 * @param pVCpu Pointer to the VMCPU.
321 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
322 * out-of-sync. Make sure to update the required
323 * fields before using them.
324 * @param pVmxTransient Pointer to the VMX-transient structure.
325 */
326#ifndef HMVMX_USE_FUNCTION_TABLE
327typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328#else
329typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330/** Pointer to VM-exit handler. */
331typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
332#endif
333
334
335/*******************************************************************************
336* Internal Functions *
337*******************************************************************************/
338static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
339static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
340static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
341 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
342#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
343static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
344#endif
345#ifndef HMVMX_USE_FUNCTION_TABLE
346DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
347# define HMVMX_EXIT_DECL static int
348#else
349# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
350#endif
351
352/** @name VM-exit handlers.
353 * @{
354 */
355static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
356static FNVMXEXITHANDLER hmR0VmxExitExtInt;
357static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
358static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
359static FNVMXEXITHANDLER hmR0VmxExitSipi;
360static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
361static FNVMXEXITHANDLER hmR0VmxExitSmi;
362static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
363static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
364static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
365static FNVMXEXITHANDLER hmR0VmxExitCpuid;
366static FNVMXEXITHANDLER hmR0VmxExitGetsec;
367static FNVMXEXITHANDLER hmR0VmxExitHlt;
368static FNVMXEXITHANDLER hmR0VmxExitInvd;
369static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
370static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
371static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
372static FNVMXEXITHANDLER hmR0VmxExitRsm;
373static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
374static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
375static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
376static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
377static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
378static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
379static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
380static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
381static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
382static FNVMXEXITHANDLER hmR0VmxExitMwait;
383static FNVMXEXITHANDLER hmR0VmxExitMtf;
384static FNVMXEXITHANDLER hmR0VmxExitMonitor;
385static FNVMXEXITHANDLER hmR0VmxExitPause;
386static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
387static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
388static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
389static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
391static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
392static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
393static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
394static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
395static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
396static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
397static FNVMXEXITHANDLER hmR0VmxExitRdrand;
398static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
399/** @} */
400
401static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
408static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409#endif
410static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
411
412/*******************************************************************************
413* Global Variables *
414*******************************************************************************/
415#ifdef HMVMX_USE_FUNCTION_TABLE
416
417/**
418 * VMX_EXIT dispatch table.
419 */
420static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
421{
422 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
423 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
424 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
425 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
426 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
427 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
428 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
429 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
430 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
431 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
432 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
433 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
434 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
435 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
436 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
437 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
438 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
439 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
440 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
441 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
442 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
443 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
444 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
445 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
446 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
447 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
448 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
449 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
450 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
451 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
452 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
453 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
454 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
455 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
456 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
457 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
458 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
459 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
460 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
461 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
462 /* 40 UNDEFINED */ hmR0VmxExitPause,
463 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
464 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
465 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
466 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
467 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
468 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
469 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
470 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
471 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
472 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
473 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
474 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
475 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
476 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
477 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
478 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
479 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
480 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
481 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
482};
483#endif /* HMVMX_USE_FUNCTION_TABLE */
484
485#ifdef VBOX_STRICT
486static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
487{
488 /* 0 */ "(Not Used)",
489 /* 1 */ "VMCALL executed in VMX root operation.",
490 /* 2 */ "VMCLEAR with invalid physical address.",
491 /* 3 */ "VMCLEAR with VMXON pointer.",
492 /* 4 */ "VMLAUNCH with non-clear VMCS.",
493 /* 5 */ "VMRESUME with non-launched VMCS.",
494 /* 6 */ "VMRESUME after VMXOFF",
495 /* 7 */ "VM entry with invalid control fields.",
496 /* 8 */ "VM entry with invalid host state fields.",
497 /* 9 */ "VMPTRLD with invalid physical address.",
498 /* 10 */ "VMPTRLD with VMXON pointer.",
499 /* 11 */ "VMPTRLD with incorrect revision identifier.",
500 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
501 /* 13 */ "VMWRITE to read-only VMCS component.",
502 /* 14 */ "(Not Used)",
503 /* 15 */ "VMXON executed in VMX root operation.",
504 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
505 /* 17 */ "VM entry with non-launched executing VMCS.",
506 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
507 /* 19 */ "VMCALL with non-clear VMCS.",
508 /* 20 */ "VMCALL with invalid VM-exit control fields.",
509 /* 21 */ "(Not Used)",
510 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
511 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
512 /* 24 */ "VMCALL with invalid SMM-monitor features.",
513 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
514 /* 26 */ "VM entry with events blocked by MOV SS.",
515 /* 27 */ "(Not Used)",
516 /* 28 */ "Invalid operand to INVEPT/INVVPID."
517};
518#endif /* VBOX_STRICT */
519
520
521
522/**
523 * Updates the VM's last error record. If there was a VMX instruction error,
524 * reads the error data from the VMCS and updates VCPU's last error record as
525 * well.
526 *
527 * @param pVM Pointer to the VM.
528 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
529 * VERR_VMX_UNABLE_TO_START_VM or
530 * VERR_VMX_INVALID_VMCS_FIELD).
531 * @param rc The error code.
532 */
533static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
534{
535 AssertPtr(pVM);
536 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
537 || rc == VERR_VMX_UNABLE_TO_START_VM)
538 {
539 AssertPtrReturnVoid(pVCpu);
540 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
541 }
542 pVM->hm.s.lLastError = rc;
543}
544
545
546/**
547 * Reads the VM-entry interruption-information field from the VMCS into the VMX
548 * transient structure.
549 *
550 * @returns VBox status code.
551 * @param pVmxTransient Pointer to the VMX transient structure.
552 *
553 * @remarks No-long-jump zone!!!
554 */
555DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
556{
557 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
558 AssertRCReturn(rc, rc);
559 return VINF_SUCCESS;
560}
561
562
563/**
564 * Reads the VM-entry exception error code field from the VMCS into
565 * the VMX transient structure.
566 *
567 * @returns VBox status code.
568 * @param pVmxTransient Pointer to the VMX transient structure.
569 *
570 * @remarks No-long-jump zone!!!
571 */
572DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
573{
574 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
575 AssertRCReturn(rc, rc);
576 return VINF_SUCCESS;
577}
578
579
580/**
581 * Reads the VM-entry exception error code field from the VMCS into
582 * the VMX transient structure.
583 *
584 * @returns VBox status code.
585 * @param pVmxTransient Pointer to the VMX transient structure.
586 *
587 * @remarks No-long-jump zone!!!
588 */
589DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
590{
591 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
592 AssertRCReturn(rc, rc);
593 return VINF_SUCCESS;
594}
595
596
597/**
598 * Reads the VM-exit interruption-information field from the VMCS into the VMX
599 * transient structure.
600 *
601 * @returns VBox status code.
602 * @param pVmxTransient Pointer to the VMX transient structure.
603 */
604DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
605{
606 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
607 {
608 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
609 AssertRCReturn(rc, rc);
610 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
611 }
612 return VINF_SUCCESS;
613}
614
615
616/**
617 * Reads the VM-exit interruption error code from the VMCS into the VMX
618 * transient structure.
619 *
620 * @returns VBox status code.
621 * @param pVmxTransient Pointer to the VMX transient structure.
622 */
623DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624{
625 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
626 {
627 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
628 AssertRCReturn(rc, rc);
629 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
630 }
631 return VINF_SUCCESS;
632}
633
634
635/**
636 * Reads the VM-exit instruction length field from the VMCS into the VMX
637 * transient structure.
638 *
639 * @returns VBox status code.
640 * @param pVCpu Pointer to the VMCPU.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit instruction-information field from the VMCS into
657 * the VMX transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the exit qualification from the VMCS into the VMX transient structure.
676 *
677 * @returns VBox status code.
678 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
679 * case).
680 * @param pVmxTransient Pointer to the VMX transient structure.
681 */
682DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
685 {
686 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the IDT-vectoring information field from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 *
701 * @remarks No-long-jump zone!!!
702 */
703DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
704{
705 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
706 {
707 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
708 AssertRCReturn(rc, rc);
709 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
710 }
711 return VINF_SUCCESS;
712}
713
714
715/**
716 * Reads the IDT-vectoring error code from the VMCS into the VMX
717 * transient structure.
718 *
719 * @returns VBox status code.
720 * @param pVmxTransient Pointer to the VMX transient structure.
721 */
722DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
723{
724 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
725 {
726 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
727 AssertRCReturn(rc, rc);
728 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
729 }
730 return VINF_SUCCESS;
731}
732
733
734/**
735 * Enters VMX root mode operation on the current CPU.
736 *
737 * @returns VBox status code.
738 * @param pVM Pointer to the VM (optional, can be NULL, after
739 * a resume).
740 * @param HCPhysCpuPage Physical address of the VMXON region.
741 * @param pvCpuPage Pointer to the VMXON region.
742 */
743static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
744{
745 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
746 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
747 Assert(pvCpuPage);
748 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
749
750 if (pVM)
751 {
752 /* Write the VMCS revision dword to the VMXON region. */
753 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
754 }
755
756 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
757 RTCCUINTREG uEflags = ASMIntDisableFlags();
758
759 /* Enable the VMX bit in CR4 if necessary. */
760 RTCCUINTREG uCr4 = ASMGetCR4();
761 if (!(uCr4 & X86_CR4_VMXE))
762 ASMSetCR4(uCr4 | X86_CR4_VMXE);
763
764 /* Enter VMX root mode. */
765 int rc = VMXEnable(HCPhysCpuPage);
766 if (RT_FAILURE(rc))
767 ASMSetCR4(uCr4);
768
769 /* Restore interrupts. */
770 ASMSetFlags(uEflags);
771 return rc;
772}
773
774
775/**
776 * Exits VMX root mode operation on the current CPU.
777 *
778 * @returns VBox status code.
779 */
780static int hmR0VmxLeaveRootMode(void)
781{
782 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
783
784 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
785 RTCCUINTREG uEflags = ASMIntDisableFlags();
786
787 /* If we're for some reason not in VMX root mode, then don't leave it. */
788 RTCCUINTREG uHostCR4 = ASMGetCR4();
789
790 int rc;
791 if (uHostCR4 & X86_CR4_VMXE)
792 {
793 /* Exit VMX root mode and clear the VMX bit in CR4. */
794 VMXDisable();
795 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
796 rc = VINF_SUCCESS;
797 }
798 else
799 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
800
801 /* Restore interrupts. */
802 ASMSetFlags(uEflags);
803 return rc;
804}
805
806
807/**
808 * Allocates and maps one physically contiguous page. The allocated page is
809 * zero'd out. (Used by various VT-x structures).
810 *
811 * @returns IPRT status code.
812 * @param pMemObj Pointer to the ring-0 memory object.
813 * @param ppVirt Where to store the virtual address of the
814 * allocation.
815 * @param pPhys Where to store the physical address of the
816 * allocation.
817 */
818DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
819{
820 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
821 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
822 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
823
824 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
825 if (RT_FAILURE(rc))
826 return rc;
827 *ppVirt = RTR0MemObjAddress(*pMemObj);
828 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
829 ASMMemZero32(*ppVirt, PAGE_SIZE);
830 return VINF_SUCCESS;
831}
832
833
834/**
835 * Frees and unmaps an allocated physical page.
836 *
837 * @param pMemObj Pointer to the ring-0 memory object.
838 * @param ppVirt Where to re-initialize the virtual address of
839 * allocation as 0.
840 * @param pHCPhys Where to re-initialize the physical address of the
841 * allocation as 0.
842 */
843DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
844{
845 AssertPtr(pMemObj);
846 AssertPtr(ppVirt);
847 AssertPtr(pHCPhys);
848 if (*pMemObj != NIL_RTR0MEMOBJ)
849 {
850 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
851 AssertRC(rc);
852 *pMemObj = NIL_RTR0MEMOBJ;
853 *ppVirt = 0;
854 *pHCPhys = 0;
855 }
856}
857
858
859/**
860 * Worker function to free VT-x related structures.
861 *
862 * @returns IPRT status code.
863 * @param pVM Pointer to the VM.
864 */
865static void hmR0VmxStructsFree(PVM pVM)
866{
867 for (VMCPUID i = 0; i < pVM->cCpus; i++)
868 {
869 PVMCPU pVCpu = &pVM->aCpus[i];
870 AssertPtr(pVCpu);
871
872 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
873 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
874
875 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
877
878 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
879 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
880 }
881
882 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
883#ifdef VBOX_WITH_CRASHDUMP_MAGIC
884 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
885#endif
886}
887
888
889/**
890 * Worker function to allocate VT-x related VM structures.
891 *
892 * @returns IPRT status code.
893 * @param pVM Pointer to the VM.
894 */
895static int hmR0VmxStructsAlloc(PVM pVM)
896{
897 /*
898 * Initialize members up-front so we can cleanup properly on allocation failure.
899 */
900#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
901 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
902 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
903 pVM->hm.s.vmx.HCPhys##a_Name = 0;
904
905#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
906 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
907 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
908 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
909
910#ifdef VBOX_WITH_CRASHDUMP_MAGIC
911 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
912#endif
913 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
914
915 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
916 for (VMCPUID i = 0; i < pVM->cCpus; i++)
917 {
918 PVMCPU pVCpu = &pVM->aCpus[i];
919 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
920 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
924 }
925#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
926#undef VMXLOCAL_INIT_VM_MEMOBJ
927
928 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
929 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
930 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
931 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
932
933 /*
934 * Allocate all the VT-x structures.
935 */
936 int rc = VINF_SUCCESS;
937#ifdef VBOX_WITH_CRASHDUMP_MAGIC
938 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
939 if (RT_FAILURE(rc))
940 goto cleanup;
941 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
942 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
943#endif
944
945 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
946 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
947 {
948 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
949 &pVM->hm.s.vmx.HCPhysApicAccess);
950 if (RT_FAILURE(rc))
951 goto cleanup;
952 }
953
954 /*
955 * Initialize per-VCPU VT-x structures.
956 */
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 AssertPtr(pVCpu);
961
962 /* Allocate the VM control structure (VMCS). */
963 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
964 if (RT_FAILURE(rc))
965 goto cleanup;
966
967 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
968 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
969 {
970 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
971 &pVCpu->hm.s.vmx.HCPhysVirtApic);
972 if (RT_FAILURE(rc))
973 goto cleanup;
974 }
975
976 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
977 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
978 {
979 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
980 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
981 if (RT_FAILURE(rc))
982 goto cleanup;
983 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
984 }
985
986 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
987 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
988 if (RT_FAILURE(rc))
989 goto cleanup;
990
991 /* Allocate the VM-exit MSR-load page for the host MSRs. */
992 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 }
996
997 return VINF_SUCCESS;
998
999cleanup:
1000 hmR0VmxStructsFree(pVM);
1001 return rc;
1002}
1003
1004
1005/**
1006 * Does global VT-x initialization (called during module initialization).
1007 *
1008 * @returns VBox status code.
1009 */
1010VMMR0DECL(int) VMXR0GlobalInit(void)
1011{
1012#ifdef HMVMX_USE_FUNCTION_TABLE
1013 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1014# ifdef VBOX_STRICT
1015 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1016 Assert(g_apfnVMExitHandlers[i]);
1017# endif
1018#endif
1019 return VINF_SUCCESS;
1020}
1021
1022
1023/**
1024 * Does global VT-x termination (called during module termination).
1025 */
1026VMMR0DECL(void) VMXR0GlobalTerm()
1027{
1028 /* Nothing to do currently. */
1029}
1030
1031
1032/**
1033 * Sets up and activates VT-x on the current CPU.
1034 *
1035 * @returns VBox status code.
1036 * @param pCpu Pointer to the global CPU info struct.
1037 * @param pVM Pointer to the VM (can be NULL after a host resume
1038 * operation).
1039 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1040 * fEnabledByHost is true).
1041 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1042 * @a fEnabledByHost is true).
1043 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1044 * enable VT-x on the host.
1045 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1046 */
1047VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1048 void *pvMsrs)
1049{
1050 Assert(pCpu);
1051 Assert(pvMsrs);
1052 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1053
1054 /* Enable VT-x if it's not already enabled by the host. */
1055 if (!fEnabledByHost)
1056 {
1057 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1058 if (RT_FAILURE(rc))
1059 return rc;
1060 }
1061
1062 /*
1063 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1064 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1065 */
1066 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1067 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1068 {
1069 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1070 pCpu->fFlushAsidBeforeUse = false;
1071 }
1072 else
1073 pCpu->fFlushAsidBeforeUse = true;
1074
1075 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1076 ++pCpu->cTlbFlushes;
1077
1078 return VINF_SUCCESS;
1079}
1080
1081
1082/**
1083 * Deactivates VT-x on the current CPU.
1084 *
1085 * @returns VBox status code.
1086 * @param pCpu Pointer to the global CPU info struct.
1087 * @param pvCpuPage Pointer to the VMXON region.
1088 * @param HCPhysCpuPage Physical address of the VMXON region.
1089 *
1090 * @remarks This function should never be called when SUPR0EnableVTx() or
1091 * similar was used to enable VT-x on the host.
1092 */
1093VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1094{
1095 NOREF(pCpu);
1096 NOREF(pvCpuPage);
1097 NOREF(HCPhysCpuPage);
1098
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100 return hmR0VmxLeaveRootMode();
1101}
1102
1103
1104/**
1105 * Sets the permission bits for the specified MSR in the MSR bitmap.
1106 *
1107 * @param pVCpu Pointer to the VMCPU.
1108 * @param uMSR The MSR value.
1109 * @param enmRead Whether reading this MSR causes a VM-exit.
1110 * @param enmWrite Whether writing this MSR causes a VM-exit.
1111 */
1112static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1113{
1114 int32_t iBit;
1115 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1116
1117 /*
1118 * Layout:
1119 * 0x000 - 0x3ff - Low MSR read bits
1120 * 0x400 - 0x7ff - High MSR read bits
1121 * 0x800 - 0xbff - Low MSR write bits
1122 * 0xc00 - 0xfff - High MSR write bits
1123 */
1124 if (uMsr <= 0x00001FFF)
1125 iBit = uMsr;
1126 else if ( uMsr >= 0xC0000000
1127 && uMsr <= 0xC0001FFF)
1128 {
1129 iBit = (uMsr - 0xC0000000);
1130 pbMsrBitmap += 0x400;
1131 }
1132 else
1133 {
1134 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1135 return;
1136 }
1137
1138 Assert(iBit <= 0x1fff);
1139 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1140 ASMBitSet(pbMsrBitmap, iBit);
1141 else
1142 ASMBitClear(pbMsrBitmap, iBit);
1143
1144 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1145 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1146 else
1147 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1148}
1149
1150
1151#ifdef VBOX_STRICT
1152/**
1153 * Gets the permission bits for the specified MSR in the MSR bitmap.
1154 *
1155 * @returns VBox status code.
1156 * @retval VINF_SUCCESS if the specified MSR is found.
1157 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1158 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1159 *
1160 * @param pVCpu Pointer to the VMCPU.
1161 * @param uMsr The MSR.
1162 * @param penmRead Where to store the read permissions.
1163 * @param penmWrite Where to store the write permissions.
1164 */
1165static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1166{
1167 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1168 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1169 int32_t iBit;
1170 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1171
1172 /* See hmR0VmxSetMsrPermission() for the layout. */
1173 if (uMsr <= 0x00001FFF)
1174 iBit = uMsr;
1175 else if ( uMsr >= 0xC0000000
1176 && uMsr <= 0xC0001FFF)
1177 {
1178 iBit = (uMsr - 0xC0000000);
1179 pbMsrBitmap += 0x400;
1180 }
1181 else
1182 {
1183 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1184 return VERR_NOT_SUPPORTED;
1185 }
1186
1187 Assert(iBit <= 0x1fff);
1188 if (ASMBitTest(pbMsrBitmap, iBit))
1189 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1190 else
1191 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1192
1193 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1194 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1195 else
1196 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1197 return VINF_SUCCESS;
1198}
1199#endif /* VBOX_STRICT */
1200
1201
1202/**
1203 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1204 * area.
1205 *
1206 * @returns VBox status code.
1207 * @param pVCpu Pointer to the VMCPU.
1208 * @param cMsrs The number of MSRs.
1209 */
1210DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1211{
1212 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1213 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1214 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1215 {
1216 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1217 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1218 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1219 }
1220
1221 /* Update number of guest MSRs to load/store across the world-switch. */
1222 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1224
1225 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1226 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1227
1228 /* Update the VCPU's copy of the MSR count. */
1229 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1230
1231 return VINF_SUCCESS;
1232}
1233
1234
1235/**
1236 * Adds a new (or updates the value of an existing) guest/host MSR
1237 * pair to be swapped during the world-switch as part of the
1238 * auto-load/store MSR area in the VMCS.
1239 *
1240 * @returns true if the MSR was added -and- its value was updated, false
1241 * otherwise.
1242 * @param pVCpu Pointer to the VMCPU.
1243 * @param uMsr The MSR.
1244 * @param uGuestMsr Value of the guest MSR.
1245 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1246 * necessary.
1247 */
1248static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1249{
1250 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1251 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1252 uint32_t i;
1253 for (i = 0; i < cMsrs; i++)
1254 {
1255 if (pGuestMsr->u32Msr == uMsr)
1256 break;
1257 pGuestMsr++;
1258 }
1259
1260 bool fAdded = false;
1261 if (i == cMsrs)
1262 {
1263 ++cMsrs;
1264 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1265 AssertRC(rc);
1266
1267 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1268 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1269 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1270
1271 fAdded = true;
1272 }
1273
1274 /* Update the MSR values in the auto-load/store MSR area. */
1275 pGuestMsr->u32Msr = uMsr;
1276 pGuestMsr->u64Value = uGuestMsrValue;
1277
1278 /* Create/update the MSR slot in the host MSR area. */
1279 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1280 pHostMsr += i;
1281 pHostMsr->u32Msr = uMsr;
1282
1283 /*
1284 * Update the host MSR only when requested by the caller AND when we're
1285 * adding it to the auto-load/store area. Otherwise, it would have been
1286 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1287 */
1288 bool fUpdatedMsrValue = false;
1289 if ( fAdded
1290 && fUpdateHostMsr)
1291 {
1292 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1293 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1294 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1295 fUpdatedMsrValue = true;
1296 }
1297
1298 return fUpdatedMsrValue;
1299}
1300
1301
1302/**
1303 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1304 * auto-load/store MSR area in the VMCS.
1305 *
1306 * @returns VBox status code.
1307 * @param pVCpu Pointer to the VMCPU.
1308 * @param uMsr The MSR.
1309 */
1310static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1311{
1312 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1313 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1314 for (uint32_t i = 0; i < cMsrs; i++)
1315 {
1316 /* Find the MSR. */
1317 if (pGuestMsr->u32Msr == uMsr)
1318 {
1319 /* If it's the last MSR, simply reduce the count. */
1320 if (i == cMsrs - 1)
1321 {
1322 --cMsrs;
1323 break;
1324 }
1325
1326 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1327 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1328 pLastGuestMsr += cMsrs - 1;
1329 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1330 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1331
1332 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1333 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1334 pLastHostMsr += cMsrs - 1;
1335 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1336 pHostMsr->u64Value = pLastHostMsr->u64Value;
1337 --cMsrs;
1338 break;
1339 }
1340 pGuestMsr++;
1341 }
1342
1343 /* Update the VMCS if the count changed (meaning the MSR was found). */
1344 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1345 {
1346 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1347 AssertRCReturn(rc, rc);
1348
1349 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1350 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1351 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1352
1353 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1354 return VINF_SUCCESS;
1355 }
1356
1357 return VERR_NOT_FOUND;
1358}
1359
1360
1361/**
1362 * Checks if the specified guest MSR is part of the auto-load/store area in
1363 * the VMCS.
1364 *
1365 * @returns true if found, false otherwise.
1366 * @param pVCpu Pointer to the VMCPU.
1367 * @param uMsr The MSR to find.
1368 */
1369static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1370{
1371 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1372 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1373
1374 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1375 {
1376 if (pGuestMsr->u32Msr == uMsr)
1377 return true;
1378 }
1379 return false;
1380}
1381
1382
1383/**
1384 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1385 *
1386 * @param pVCpu Pointer to the VMCPU.
1387 *
1388 * @remarks No-long-jump zone!!!
1389 */
1390static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1391{
1392 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1393 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1394 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1395 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1396
1397 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1398 {
1399 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1400
1401 /*
1402 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1403 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1404 */
1405 if (pHostMsr->u32Msr == MSR_K6_EFER)
1406 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1407 else
1408 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1409 }
1410
1411 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1412}
1413
1414
1415#if HC_ARCH_BITS == 64
1416/**
1417 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1418 * perform lazy restoration of the host MSRs while leaving VT-x.
1419 *
1420 * @param pVCpu Pointer to the VMCPU.
1421 *
1422 * @remarks No-long-jump zone!!!
1423 */
1424static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1425{
1426 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1427
1428 /*
1429 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1430 */
1431 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1432 {
1433 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1434 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1435 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1436 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1437 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1438 }
1439}
1440
1441
1442/**
1443 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1444 * lazily while leaving VT-x.
1445 *
1446 * @returns true if it does, false otherwise.
1447 * @param pVCpu Pointer to the VMCPU.
1448 * @param uMsr The MSR to check.
1449 */
1450static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1451{
1452 NOREF(pVCpu);
1453 switch (uMsr)
1454 {
1455 case MSR_K8_LSTAR:
1456 case MSR_K6_STAR:
1457 case MSR_K8_SF_MASK:
1458 case MSR_K8_KERNEL_GS_BASE:
1459 return true;
1460 }
1461 return false;
1462}
1463
1464
1465/**
1466 * Saves a set of guests MSRs back into the guest-CPU context.
1467 *
1468 * @param pVCpu Pointer to the VMCPU.
1469 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1470 * out-of-sync. Make sure to update the required fields
1471 * before using them.
1472 *
1473 * @remarks No-long-jump zone!!!
1474 */
1475static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1476{
1477 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1478 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1479
1480 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1481 {
1482 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1483 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1484 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1485 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1486 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487 }
1488}
1489
1490
1491/**
1492 * Loads a set of guests MSRs to allow read/passthru to the guest.
1493 *
1494 * The name of this function is slightly confusing. This function does NOT
1495 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1496 * common prefix for functions dealing with "lazy restoration" of the shared
1497 * MSRs.
1498 *
1499 * @param pVCpu Pointer to the VMCPU.
1500 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1501 * out-of-sync. Make sure to update the required fields
1502 * before using them.
1503 *
1504 * @remarks No-long-jump zone!!!
1505 */
1506static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1507{
1508 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1509 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1510
1511 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1512 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1513 {
1514#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1515 do { \
1516 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1517 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1518 else \
1519 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1520 } while (0)
1521
1522 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1523 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1524 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1525 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1526#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1527 }
1528 else
1529 {
1530 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1531 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1532 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1533 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1534 }
1535 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1536}
1537
1538
1539/**
1540 * Performs lazy restoration of the set of host MSRs if they were previously
1541 * loaded with guest MSR values.
1542 *
1543 * @param pVCpu Pointer to the VMCPU.
1544 *
1545 * @remarks No-long-jump zone!!!
1546 * @remarks The guest MSRs should have been saved back into the guest-CPU
1547 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1548 */
1549static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1550{
1551 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1552 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1553
1554 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1555 {
1556 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1557 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1558 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1559 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1560 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1561 }
1562 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1563}
1564#endif /* HC_ARCH_BITS == 64 */
1565
1566
1567/**
1568 * Verifies that our cached values of the VMCS controls are all
1569 * consistent with what's actually present in the VMCS.
1570 *
1571 * @returns VBox status code.
1572 * @param pVCpu Pointer to the VMCPU.
1573 */
1574static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1575{
1576 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1577
1578 uint32_t u32Val;
1579 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1580 AssertRCReturn(rc, rc);
1581 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1582 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1583
1584 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1585 AssertRCReturn(rc, rc);
1586 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1587 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1588
1589 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1590 AssertRCReturn(rc, rc);
1591 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1592 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1593
1594 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1595 AssertRCReturn(rc, rc);
1596 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1597 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1598
1599 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1600 AssertRCReturn(rc, rc);
1601 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1602 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1603
1604 return VINF_SUCCESS;
1605}
1606
1607
1608#ifdef VBOX_STRICT
1609/**
1610 * Verifies that our cached host EFER value has not changed
1611 * since we cached it.
1612 *
1613 * @param pVCpu Pointer to the VMCPU.
1614 */
1615static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1616{
1617 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1618
1619 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1620 {
1621 uint64_t u64Val;
1622 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1623 AssertRC(rc);
1624
1625 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1626 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1627 }
1628}
1629
1630
1631/**
1632 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1633 * VMCS are correct.
1634 *
1635 * @param pVCpu Pointer to the VMCPU.
1636 */
1637static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1638{
1639 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1640
1641 /* Verify MSR counts in the VMCS are what we think it should be. */
1642 uint32_t cMsrs;
1643 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1644 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1645
1646 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1647 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1648
1649 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1650 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1651
1652 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1653 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1654 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1655 {
1656 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1657 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1658 pGuestMsr->u32Msr, cMsrs));
1659
1660 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1661 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1662 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1663
1664 /* Verify that the permissions are as expected in the MSR bitmap. */
1665 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1666 {
1667 VMXMSREXITREAD enmRead;
1668 VMXMSREXITWRITE enmWrite;
1669 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1670 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1671 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1672 {
1673 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1674 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1675 }
1676 else
1677 {
1678 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1679 pGuestMsr->u32Msr, cMsrs));
1680 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1681 pGuestMsr->u32Msr, cMsrs));
1682 }
1683 }
1684 }
1685}
1686#endif /* VBOX_STRICT */
1687
1688
1689/**
1690 * Flushes the TLB using EPT.
1691 *
1692 * @returns VBox status code.
1693 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1694 * enmFlush).
1695 * @param enmFlush Type of flush.
1696 *
1697 * @remarks Caller is responsible for making sure this function is called only
1698 * when NestedPaging is supported and providing @a enmFlush that is
1699 * supported by the CPU.
1700 * @remarks Can be called with interrupts disabled.
1701 */
1702static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1703{
1704 uint64_t au64Descriptor[2];
1705 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1706 au64Descriptor[0] = 0;
1707 else
1708 {
1709 Assert(pVCpu);
1710 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1711 }
1712 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1713
1714 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1715 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1716 rc));
1717 if ( RT_SUCCESS(rc)
1718 && pVCpu)
1719 {
1720 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1721 }
1722}
1723
1724
1725/**
1726 * Flushes the TLB using VPID.
1727 *
1728 * @returns VBox status code.
1729 * @param pVM Pointer to the VM.
1730 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1731 * enmFlush).
1732 * @param enmFlush Type of flush.
1733 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1734 * on @a enmFlush).
1735 *
1736 * @remarks Can be called with interrupts disabled.
1737 */
1738static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1739{
1740 NOREF(pVM);
1741 AssertPtr(pVM);
1742 Assert(pVM->hm.s.vmx.fVpid);
1743
1744 uint64_t au64Descriptor[2];
1745 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1746 {
1747 au64Descriptor[0] = 0;
1748 au64Descriptor[1] = 0;
1749 }
1750 else
1751 {
1752 AssertPtr(pVCpu);
1753 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1754 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1755 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1756 au64Descriptor[1] = GCPtr;
1757 }
1758
1759 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1760 AssertMsg(rc == VINF_SUCCESS,
1761 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1762 if ( RT_SUCCESS(rc)
1763 && pVCpu)
1764 {
1765 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1766 }
1767}
1768
1769
1770/**
1771 * Invalidates a guest page by guest virtual address. Only relevant for
1772 * EPT/VPID, otherwise there is nothing really to invalidate.
1773 *
1774 * @returns VBox status code.
1775 * @param pVM Pointer to the VM.
1776 * @param pVCpu Pointer to the VMCPU.
1777 * @param GCVirt Guest virtual address of the page to invalidate.
1778 */
1779VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1780{
1781 AssertPtr(pVM);
1782 AssertPtr(pVCpu);
1783 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1784
1785 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1786 if (!fFlushPending)
1787 {
1788 /*
1789 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1790 * See @bugref{6043} and @bugref{6177}.
1791 *
1792 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1793 * function maybe called in a loop with individual addresses.
1794 */
1795 if (pVM->hm.s.vmx.fVpid)
1796 {
1797 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1798 {
1799 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1800 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1801 }
1802 else
1803 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1804 }
1805 else if (pVM->hm.s.fNestedPaging)
1806 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1807 }
1808
1809 return VINF_SUCCESS;
1810}
1811
1812
1813/**
1814 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1815 * otherwise there is nothing really to invalidate.
1816 *
1817 * @returns VBox status code.
1818 * @param pVM Pointer to the VM.
1819 * @param pVCpu Pointer to the VMCPU.
1820 * @param GCPhys Guest physical address of the page to invalidate.
1821 */
1822VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1823{
1824 NOREF(pVM); NOREF(GCPhys);
1825 LogFlowFunc(("%RGp\n", GCPhys));
1826
1827 /*
1828 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1829 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1830 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1831 */
1832 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1833 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1834 return VINF_SUCCESS;
1835}
1836
1837
1838/**
1839 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1840 * case where neither EPT nor VPID is supported by the CPU.
1841 *
1842 * @param pVM Pointer to the VM.
1843 * @param pVCpu Pointer to the VMCPU.
1844 * @param pCpu Pointer to the global HM struct.
1845 *
1846 * @remarks Called with interrupts disabled.
1847 */
1848static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1849{
1850 AssertPtr(pVCpu);
1851 AssertPtr(pCpu);
1852 NOREF(pVM);
1853
1854 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1855
1856 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1857#if 0
1858 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1859 pVCpu->hm.s.TlbShootdown.cPages = 0;
1860#endif
1861
1862 Assert(pCpu->idCpu != NIL_RTCPUID);
1863 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1864 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1865 pVCpu->hm.s.fForceTLBFlush = false;
1866 return;
1867}
1868
1869
1870/**
1871 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1872 *
1873 * @param pVM Pointer to the VM.
1874 * @param pVCpu Pointer to the VMCPU.
1875 * @param pCpu Pointer to the global HM CPU struct.
1876 * @remarks All references to "ASID" in this function pertains to "VPID" in
1877 * Intel's nomenclature. The reason is, to avoid confusion in compare
1878 * statements since the host-CPU copies are named "ASID".
1879 *
1880 * @remarks Called with interrupts disabled.
1881 */
1882static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1883{
1884#ifdef VBOX_WITH_STATISTICS
1885 bool fTlbFlushed = false;
1886# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1887# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1888 if (!fTlbFlushed) \
1889 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1890 } while (0)
1891#else
1892# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1893# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1894#endif
1895
1896 AssertPtr(pVM);
1897 AssertPtr(pCpu);
1898 AssertPtr(pVCpu);
1899 Assert(pCpu->idCpu != NIL_RTCPUID);
1900
1901 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1902 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1903 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1904
1905 /*
1906 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1907 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1908 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1909 */
1910 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1911 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1912 {
1913 ++pCpu->uCurrentAsid;
1914 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1915 {
1916 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1917 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1918 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1919 }
1920
1921 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924
1925 /*
1926 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1927 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1928 */
1929 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1930 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1931 HMVMX_SET_TAGGED_TLB_FLUSHED();
1932 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1933 }
1934
1935 /* Check for explicit TLB shootdowns. */
1936 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1937 {
1938 /*
1939 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1940 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1941 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1942 * but not guest-physical mappings.
1943 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1944 */
1945 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1947 HMVMX_SET_TAGGED_TLB_FLUSHED();
1948 }
1949
1950 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1951 * where it is commented out. Support individual entry flushing
1952 * someday. */
1953#if 0
1954 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1955 {
1956 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1957
1958 /*
1959 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1960 * as supported by the CPU.
1961 */
1962 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1963 {
1964 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1965 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1966 }
1967 else
1968 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1969
1970 HMVMX_SET_TAGGED_TLB_FLUSHED();
1971 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972 pVCpu->hm.s.TlbShootdown.cPages = 0;
1973 }
1974#endif
1975
1976 pVCpu->hm.s.fForceTLBFlush = false;
1977
1978 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1979
1980 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1981 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1982 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1983 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1984 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1985 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1986 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1987 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1988 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1989
1990 /* Update VMCS with the VPID. */
1991 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1992 AssertRC(rc);
1993
1994#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1995}
1996
1997
1998/**
1999 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2000 *
2001 * @returns VBox status code.
2002 * @param pVM Pointer to the VM.
2003 * @param pVCpu Pointer to the VMCPU.
2004 * @param pCpu Pointer to the global HM CPU struct.
2005 *
2006 * @remarks Called with interrupts disabled.
2007 */
2008static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2009{
2010 AssertPtr(pVM);
2011 AssertPtr(pVCpu);
2012 AssertPtr(pCpu);
2013 Assert(pCpu->idCpu != NIL_RTCPUID);
2014 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2015 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2016
2017 /*
2018 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2019 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2020 */
2021 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2022 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2023 {
2024 pVCpu->hm.s.fForceTLBFlush = true;
2025 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2026 }
2027
2028 /* Check for explicit TLB shootdown flushes. */
2029 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2030 {
2031 pVCpu->hm.s.fForceTLBFlush = true;
2032 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2033 }
2034
2035 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2036 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2037
2038 if (pVCpu->hm.s.fForceTLBFlush)
2039 {
2040 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2041 pVCpu->hm.s.fForceTLBFlush = false;
2042 }
2043 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2044 * where it is commented out. Support individual entry flushing
2045 * someday. */
2046#if 0
2047 else
2048 {
2049 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2050 {
2051 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2052 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2053 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2054 }
2055 else
2056 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2057
2058 pVCpu->hm.s.TlbShootdown.cPages = 0;
2059 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060 }
2061#endif
2062}
2063
2064
2065/**
2066 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2067 *
2068 * @returns VBox status code.
2069 * @param pVM Pointer to the VM.
2070 * @param pVCpu Pointer to the VMCPU.
2071 * @param pCpu Pointer to the global HM CPU struct.
2072 *
2073 * @remarks Called with interrupts disabled.
2074 */
2075static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2076{
2077 AssertPtr(pVM);
2078 AssertPtr(pVCpu);
2079 AssertPtr(pCpu);
2080 Assert(pCpu->idCpu != NIL_RTCPUID);
2081 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2082 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2083
2084 /*
2085 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2086 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2087 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2088 */
2089 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2090 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2091 {
2092 pVCpu->hm.s.fForceTLBFlush = true;
2093 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2094 }
2095
2096 /* Check for explicit TLB shootdown flushes. */
2097 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2098 {
2099 /*
2100 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2101 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2102 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2103 */
2104 pVCpu->hm.s.fForceTLBFlush = true;
2105 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2106 }
2107
2108 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2109 if (pVCpu->hm.s.fForceTLBFlush)
2110 {
2111 ++pCpu->uCurrentAsid;
2112 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2113 {
2114 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2115 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2116 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2117 }
2118
2119 pVCpu->hm.s.fForceTLBFlush = false;
2120 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2121 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2122 if (pCpu->fFlushAsidBeforeUse)
2123 {
2124 if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_SINGLE_CONTEXT)
2125 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2126 else if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_ALL_CONTEXTS)
2127 {
2128 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
2129 pCpu->fFlushAsidBeforeUse = false;
2130 }
2131 else
2132 {
2133 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2134 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2135 }
2136 }
2137 }
2138 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2139 * where it is commented out. Support individual entry flushing
2140 * someday. */
2141#if 0
2142 else
2143 {
2144 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2145 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2146 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2147 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2148
2149 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2150 {
2151 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2152 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2153 {
2154 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2155 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2156 }
2157 else
2158 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2159
2160 pVCpu->hm.s.TlbShootdown.cPages = 0;
2161 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2162 }
2163 else
2164 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2165 }
2166#endif
2167
2168 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2169 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2170 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2171 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2172 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2173 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2174 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2175
2176 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2177 AssertRC(rc);
2178}
2179
2180
2181/**
2182 * Flushes the guest TLB entry based on CPU capabilities.
2183 *
2184 * @param pVCpu Pointer to the VMCPU.
2185 * @param pCpu Pointer to the global HM CPU struct.
2186 */
2187DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2188{
2189#ifdef HMVMX_ALWAYS_FLUSH_TLB
2190 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2191#endif
2192 PVM pVM = pVCpu->CTX_SUFF(pVM);
2193 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2194 {
2195 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2196 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2197 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2198 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2199 default:
2200 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2201 break;
2202 }
2203
2204 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2205 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2206
2207 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2208}
2209
2210
2211/**
2212 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2213 * TLB entries from the host TLB before VM-entry.
2214 *
2215 * @returns VBox status code.
2216 * @param pVM Pointer to the VM.
2217 */
2218static int hmR0VmxSetupTaggedTlb(PVM pVM)
2219{
2220 /*
2221 * Determine optimal flush type for Nested Paging.
2222 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2223 * guest execution (see hmR3InitFinalizeR0()).
2224 */
2225 if (pVM->hm.s.fNestedPaging)
2226 {
2227 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2228 {
2229 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2230 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2231 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2232 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2233 else
2234 {
2235 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2236 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2237 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2238 }
2239
2240 /* Make sure the write-back cacheable memory type for EPT is supported. */
2241 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2242 {
2243 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2244 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2246 }
2247 }
2248 else
2249 {
2250 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2251 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254 }
2255
2256 /*
2257 * Determine optimal flush type for VPID.
2258 */
2259 if (pVM->hm.s.vmx.fVpid)
2260 {
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2262 {
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2264 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2265 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2266 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2267 else
2268 {
2269 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2270 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2271 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2272 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2273 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2274 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2275 pVM->hm.s.vmx.fVpid = false;
2276 }
2277 }
2278 else
2279 {
2280 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2281 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2282 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2283 pVM->hm.s.vmx.fVpid = false;
2284 }
2285 }
2286
2287 /*
2288 * Setup the handler for flushing tagged-TLBs.
2289 */
2290 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2291 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2292 else if (pVM->hm.s.fNestedPaging)
2293 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2294 else if (pVM->hm.s.vmx.fVpid)
2295 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2296 else
2297 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2298 return VINF_SUCCESS;
2299}
2300
2301
2302/**
2303 * Sets up pin-based VM-execution controls in the VMCS.
2304 *
2305 * @returns VBox status code.
2306 * @param pVM Pointer to the VM.
2307 * @param pVCpu Pointer to the VMCPU.
2308 */
2309static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2310{
2311 AssertPtr(pVM);
2312 AssertPtr(pVCpu);
2313
2314 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2315 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2316
2317 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2318 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2319 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2320
2321 /* Enable the VMX preemption timer. */
2322 if (pVM->hm.s.vmx.fUsePreemptTimer)
2323 {
2324 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2325 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2326 }
2327
2328 if ((val & zap) != val)
2329 {
2330 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2331 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2332 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2333 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2334 }
2335
2336 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2337 AssertRCReturn(rc, rc);
2338
2339 /* Update VCPU with the currently set pin-based VM-execution controls. */
2340 pVCpu->hm.s.vmx.u32PinCtls = val;
2341 return rc;
2342}
2343
2344
2345/**
2346 * Sets up processor-based VM-execution controls in the VMCS.
2347 *
2348 * @returns VBox status code.
2349 * @param pVM Pointer to the VM.
2350 * @param pVMCPU Pointer to the VMCPU.
2351 */
2352static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2353{
2354 AssertPtr(pVM);
2355 AssertPtr(pVCpu);
2356
2357 int rc = VERR_INTERNAL_ERROR_5;
2358 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2359 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2360
2361 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2362 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2363 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2364 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2365 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2368
2369 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2370 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2371 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2372 {
2373 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2374 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2375 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2376 }
2377
2378 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2379 if (!pVM->hm.s.fNestedPaging)
2380 {
2381 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2382 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2383 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2384 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2385 }
2386
2387 /* Use TPR shadowing if supported by the CPU. */
2388 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2389 {
2390 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2391 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2392 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2393 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2394 AssertRCReturn(rc, rc);
2395
2396 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2397 /* CR8 writes causes a VM-exit based on TPR threshold. */
2398 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2399 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2400 }
2401 else
2402 {
2403 /*
2404 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2405 * Set this control only for 64-bit guests.
2406 */
2407 if (pVM->hm.s.fAllow64BitGuests)
2408 {
2409 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2410 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2411 }
2412 }
2413
2414 /* Use MSR-bitmaps if supported by the CPU. */
2415 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2416 {
2417 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2418
2419 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2420 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2421 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2422 AssertRCReturn(rc, rc);
2423
2424 /*
2425 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2426 * automatically as dedicated fields in the VMCS.
2427 */
2428 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433
2434#if HC_ARCH_BITS == 64
2435 /*
2436 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2437 */
2438 if (pVM->hm.s.fAllow64BitGuests)
2439 {
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2444 }
2445#endif
2446 }
2447
2448 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2449 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2450 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2451
2452 if ((val & zap) != val)
2453 {
2454 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2455 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2456 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2457 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2458 }
2459
2460 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2461 AssertRCReturn(rc, rc);
2462
2463 /* Update VCPU with the currently set processor-based VM-execution controls. */
2464 pVCpu->hm.s.vmx.u32ProcCtls = val;
2465
2466 /*
2467 * Secondary processor-based VM-execution controls.
2468 */
2469 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2470 {
2471 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2472 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2473
2474 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2475 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2476
2477 if (pVM->hm.s.fNestedPaging)
2478 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2479 else
2480 {
2481 /*
2482 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2483 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2484 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2485 */
2486 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2487 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2488 }
2489
2490 if (pVM->hm.s.vmx.fVpid)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2492
2493 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2494 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2495
2496 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2497 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2498 * done dynamically. */
2499 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2500 {
2501 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2502 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2503 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2504 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2505 AssertRCReturn(rc, rc);
2506 }
2507
2508 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2509 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2510
2511 if ((val & zap) != val)
2512 {
2513 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2514 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2515 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2516 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2517 }
2518
2519 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2520 AssertRCReturn(rc, rc);
2521
2522 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2523 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2524 }
2525 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2526 {
2527 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2528 "available\n"));
2529 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2530 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2531 }
2532
2533 return VINF_SUCCESS;
2534}
2535
2536
2537/**
2538 * Sets up miscellaneous (everything other than Pin & Processor-based
2539 * VM-execution) control fields in the VMCS.
2540 *
2541 * @returns VBox status code.
2542 * @param pVM Pointer to the VM.
2543 * @param pVCpu Pointer to the VMCPU.
2544 */
2545static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2546{
2547 NOREF(pVM);
2548 AssertPtr(pVM);
2549 AssertPtr(pVCpu);
2550
2551 int rc = VERR_GENERAL_FAILURE;
2552
2553 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2554#if 0
2555 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2556 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2557 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2558
2559 /*
2560 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2561 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2562 * We thus use the exception bitmap to control it rather than use both.
2563 */
2564 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2565 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2566
2567 /** @todo Explore possibility of using IO-bitmaps. */
2568 /* All IO & IOIO instructions cause VM-exits. */
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2570 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2571
2572 /* Initialize the MSR-bitmap area. */
2573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2575 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2576#endif
2577
2578 /* Setup MSR auto-load/store area. */
2579 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2580 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2581 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2582 AssertRCReturn(rc, rc);
2583 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2584 AssertRCReturn(rc, rc);
2585
2586 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2587 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2588 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2589 AssertRCReturn(rc, rc);
2590
2591 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2592 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2593 AssertRCReturn(rc, rc);
2594
2595 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2596#if 0
2597 /* Setup debug controls */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2599 AssertRCReturn(rc, rc);
2600 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2601 AssertRCReturn(rc, rc);
2602#endif
2603
2604 return rc;
2605}
2606
2607
2608/**
2609 * Sets up the initial exception bitmap in the VMCS based on static conditions
2610 * (i.e. conditions that cannot ever change after starting the VM).
2611 *
2612 * @returns VBox status code.
2613 * @param pVM Pointer to the VM.
2614 * @param pVCpu Pointer to the VMCPU.
2615 */
2616static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2617{
2618 AssertPtr(pVM);
2619 AssertPtr(pVCpu);
2620
2621 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2622
2623 uint32_t u32XcptBitmap = 0;
2624
2625 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2626 if (!pVM->hm.s.fNestedPaging)
2627 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2628
2629 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2630 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2631 AssertRCReturn(rc, rc);
2632 return rc;
2633}
2634
2635
2636/**
2637 * Sets up the initial guest-state mask. The guest-state mask is consulted
2638 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2639 * for the nested virtualization case (as it would cause a VM-exit).
2640 *
2641 * @param pVCpu Pointer to the VMCPU.
2642 */
2643static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2644{
2645 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2646 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2647 return VINF_SUCCESS;
2648}
2649
2650
2651/**
2652 * Does per-VM VT-x initialization.
2653 *
2654 * @returns VBox status code.
2655 * @param pVM Pointer to the VM.
2656 */
2657VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2658{
2659 LogFlowFunc(("pVM=%p\n", pVM));
2660
2661 int rc = hmR0VmxStructsAlloc(pVM);
2662 if (RT_FAILURE(rc))
2663 {
2664 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2665 return rc;
2666 }
2667
2668 return VINF_SUCCESS;
2669}
2670
2671
2672/**
2673 * Does per-VM VT-x termination.
2674 *
2675 * @returns VBox status code.
2676 * @param pVM Pointer to the VM.
2677 */
2678VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2679{
2680 LogFlowFunc(("pVM=%p\n", pVM));
2681
2682#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2683 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2684 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2685#endif
2686 hmR0VmxStructsFree(pVM);
2687 return VINF_SUCCESS;
2688}
2689
2690
2691/**
2692 * Sets up the VM for execution under VT-x.
2693 * This function is only called once per-VM during initialization.
2694 *
2695 * @returns VBox status code.
2696 * @param pVM Pointer to the VM.
2697 */
2698VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2699{
2700 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2701 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2702
2703 LogFlowFunc(("pVM=%p\n", pVM));
2704
2705 /*
2706 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2707 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2708 */
2709 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2710 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2711 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2712 || !pVM->hm.s.vmx.pRealModeTSS))
2713 {
2714 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2715 return VERR_INTERNAL_ERROR;
2716 }
2717
2718#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719 /*
2720 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2721 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2722 */
2723 if ( pVM->hm.s.fAllow64BitGuests
2724 && !HMVMX_IS_64BIT_HOST_MODE())
2725 {
2726 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2727 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2728 }
2729#endif
2730
2731 /* Initialize these always, see hmR3InitFinalizeR0().*/
2732 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2733 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2734
2735 /* Setup the tagged-TLB flush handlers. */
2736 int rc = hmR0VmxSetupTaggedTlb(pVM);
2737 if (RT_FAILURE(rc))
2738 {
2739 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2740 return rc;
2741 }
2742
2743 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2744 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2745#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2746 if ( HMVMX_IS_64BIT_HOST_MODE()
2747 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2748 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2749 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2750 {
2751 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2752 }
2753#endif
2754
2755 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2756 {
2757 PVMCPU pVCpu = &pVM->aCpus[i];
2758 AssertPtr(pVCpu);
2759 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2760
2761 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2762 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2763
2764 /* Set revision dword at the beginning of the VMCS structure. */
2765 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2766
2767 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2768 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2769 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2770 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2771
2772 /* Load this VMCS as the current VMCS. */
2773 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2774 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2775 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2776
2777 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2779 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2780
2781 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2783 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2784
2785 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2794 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2795 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2796
2797#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2798 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801#endif
2802
2803 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2804 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2809
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2811 }
2812
2813 return VINF_SUCCESS;
2814}
2815
2816
2817/**
2818 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2819 * the VMCS.
2820 *
2821 * @returns VBox status code.
2822 * @param pVM Pointer to the VM.
2823 * @param pVCpu Pointer to the VMCPU.
2824 */
2825DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2826{
2827 NOREF(pVM); NOREF(pVCpu);
2828
2829 RTCCUINTREG uReg = ASMGetCR0();
2830 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2831 AssertRCReturn(rc, rc);
2832
2833#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2834 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2835 if (HMVMX_IS_64BIT_HOST_MODE())
2836 {
2837 uint64_t uRegCR3 = HMR0Get64bitCR3();
2838 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2839 }
2840 else
2841#endif
2842 {
2843 uReg = ASMGetCR3();
2844 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2845 }
2846 AssertRCReturn(rc, rc);
2847
2848 uReg = ASMGetCR4();
2849 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2850 AssertRCReturn(rc, rc);
2851 return rc;
2852}
2853
2854
2855#if HC_ARCH_BITS == 64
2856/**
2857 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2858 * requirements. See hmR0VmxSaveHostSegmentRegs().
2859 */
2860# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2861 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2862 { \
2863 bool fValidSelector = true; \
2864 if ((selValue) & X86_SEL_LDT) \
2865 { \
2866 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2867 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2868 } \
2869 if (fValidSelector) \
2870 { \
2871 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2872 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2873 } \
2874 (selValue) = 0; \
2875 }
2876#endif
2877
2878
2879/**
2880 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2881 * the host-state area in the VMCS.
2882 *
2883 * @returns VBox status code.
2884 * @param pVM Pointer to the VM.
2885 * @param pVCpu Pointer to the VMCPU.
2886 */
2887DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2888{
2889 NOREF(pVM);
2890 int rc = VERR_INTERNAL_ERROR_5;
2891
2892#if HC_ARCH_BITS == 64
2893 /*
2894 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2895 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2896 */
2897 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2898 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2899#endif
2900
2901 /*
2902 * Host DS, ES, FS and GS segment registers.
2903 */
2904#if HC_ARCH_BITS == 64
2905 RTSEL uSelDS = ASMGetDS();
2906 RTSEL uSelES = ASMGetES();
2907 RTSEL uSelFS = ASMGetFS();
2908 RTSEL uSelGS = ASMGetGS();
2909#else
2910 RTSEL uSelDS = 0;
2911 RTSEL uSelES = 0;
2912 RTSEL uSelFS = 0;
2913 RTSEL uSelGS = 0;
2914#endif
2915
2916 /* Recalculate which host-state bits need to be manually restored. */
2917 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2918
2919 /*
2920 * Host CS and SS segment registers.
2921 */
2922#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2923 RTSEL uSelCS;
2924 RTSEL uSelSS;
2925 if (HMVMX_IS_64BIT_HOST_MODE())
2926 {
2927 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2928 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2929 }
2930 else
2931 {
2932 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2933 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2934 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2935 }
2936#else
2937 RTSEL uSelCS = ASMGetCS();
2938 RTSEL uSelSS = ASMGetSS();
2939#endif
2940
2941 /*
2942 * Host TR segment register.
2943 */
2944 RTSEL uSelTR = ASMGetTR();
2945
2946#if HC_ARCH_BITS == 64
2947 /*
2948 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2949 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2950 */
2951 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2952 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2953 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2954 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2955# undef VMXLOCAL_ADJUST_HOST_SEG
2956#endif
2957
2958 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2959 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2960 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2961 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2962 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2963 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2964 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2965 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2966 Assert(uSelCS);
2967 Assert(uSelTR);
2968
2969 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2970#if 0
2971 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2972 Assert(uSelSS != 0);
2973#endif
2974
2975 /* Write these host selector fields into the host-state area in the VMCS. */
2976 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2977 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2978#if HC_ARCH_BITS == 64
2979 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2980 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2982 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2983#endif
2984 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2985
2986 /*
2987 * Host GDTR and IDTR.
2988 */
2989 RTGDTR Gdtr;
2990 RT_ZERO(Gdtr);
2991#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2992 if (HMVMX_IS_64BIT_HOST_MODE())
2993 {
2994 X86XDTR64 Gdtr64;
2995 X86XDTR64 Idtr64;
2996 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2997 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2999
3000 Gdtr.cbGdt = Gdtr64.cb;
3001 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3002 }
3003 else
3004#endif
3005 {
3006 RTIDTR Idtr;
3007 ASMGetGDTR(&Gdtr);
3008 ASMGetIDTR(&Idtr);
3009 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3010 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3011
3012#if HC_ARCH_BITS == 64
3013 /*
3014 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3015 * maximum limit (0xffff) on every VM-exit.
3016 */
3017 if (Gdtr.cbGdt != 0xffff)
3018 {
3019 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3020 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3021 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3022 }
3023
3024 /*
3025 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3026 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3027 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3028 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3029 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3030 * hosts where we are pretty sure it won't cause trouble.
3031 */
3032# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3033 if (Idtr.cbIdt < 0x0fff)
3034# else
3035 if (Idtr.cbIdt != 0xffff)
3036# endif
3037 {
3038 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3039 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3040 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3041 }
3042#endif
3043 }
3044
3045 /*
3046 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3047 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3048 */
3049 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3050 {
3051 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3052 return VERR_VMX_INVALID_HOST_STATE;
3053 }
3054
3055 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3056#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3057 if (HMVMX_IS_64BIT_HOST_MODE())
3058 {
3059 /* We need the 64-bit TR base for hybrid darwin. */
3060 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3061 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3062 }
3063 else
3064#endif
3065 {
3066 uintptr_t uTRBase;
3067#if HC_ARCH_BITS == 64
3068 uTRBase = X86DESC64_BASE(pDesc);
3069
3070 /*
3071 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3072 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3073 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3074 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3075 *
3076 * [1] See Intel spec. 3.5 "System Descriptor Types".
3077 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3078 */
3079 Assert(pDesc->System.u4Type == 11);
3080 if ( pDesc->System.u16LimitLow != 0x67
3081 || pDesc->System.u4LimitHigh)
3082 {
3083 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3084 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3085
3086 /* Store the GDTR here as we need it while restoring TR. */
3087 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3088 }
3089#else
3090 uTRBase = X86DESC_BASE(pDesc);
3091#endif
3092 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3093 }
3094 AssertRCReturn(rc, rc);
3095
3096 /*
3097 * Host FS base and GS base.
3098 */
3099#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3100 if (HMVMX_IS_64BIT_HOST_MODE())
3101 {
3102 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3103 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3104 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3105 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3106
3107# if HC_ARCH_BITS == 64
3108 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3109 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3110 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3111 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3112 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3113# endif
3114 }
3115#endif
3116 return rc;
3117}
3118
3119
3120/**
3121 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3122 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3123 * the host after every successful VM-exit.
3124 *
3125 * @returns VBox status code.
3126 * @param pVM Pointer to the VM.
3127 * @param pVCpu Pointer to the VMCPU.
3128 *
3129 * @remarks No-long-jump zone!!!
3130 */
3131DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3132{
3133 NOREF(pVM);
3134
3135 AssertPtr(pVCpu);
3136 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3137
3138 int rc = VINF_SUCCESS;
3139#if HC_ARCH_BITS == 64
3140 if (pVM->hm.s.fAllow64BitGuests)
3141 hmR0VmxLazySaveHostMsrs(pVCpu);
3142#endif
3143
3144 /*
3145 * Host Sysenter MSRs.
3146 */
3147 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3148 AssertRCReturn(rc, rc);
3149#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3150 if (HMVMX_IS_64BIT_HOST_MODE())
3151 {
3152 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3153 AssertRCReturn(rc, rc);
3154 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3155 }
3156 else
3157 {
3158 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3159 AssertRCReturn(rc, rc);
3160 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3161 }
3162#elif HC_ARCH_BITS == 32
3163 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3164 AssertRCReturn(rc, rc);
3165 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3166#else
3167 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3168 AssertRCReturn(rc, rc);
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3170#endif
3171 AssertRCReturn(rc, rc);
3172
3173 /*
3174 * Host EFER MSR.
3175 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3176 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3177 */
3178#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3179 if ( HMVMX_IS_64BIT_HOST_MODE()
3180 && pVM->hm.s.vmx.fSupportsVmcsEfer)
3181 {
3182 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3183 AssertRCReturn(rc, rc);
3184 }
3185#endif
3186
3187 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3188 * hmR0VmxLoadGuestExitCtls() !! */
3189
3190 return rc;
3191}
3192
3193
3194/**
3195 * Figures out if we need to swap the EFER MSR which is
3196 * particularly expensive.
3197 *
3198 * We check all relevant bits. For now, that's everything
3199 * besides LMA/LME, as these two bits are handled by VM-entry,
3200 * see hmR0VmxLoadGuestExitCtls() and
3201 * hmR0VMxLoadGuestEntryCtls().
3202 *
3203 * @returns true if we need to load guest EFER, false otherwise.
3204 * @param pVCpu Pointer to the VMCPU.
3205 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3206 * out-of-sync. Make sure to update the required fields
3207 * before using them.
3208 *
3209 * @remarks Requires EFER, CR4.
3210 * @remarks No-long-jump zone!!!
3211 */
3212static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3213{
3214#ifdef HMVMX_ALWAYS_SWAP_EFER
3215 return true;
3216#endif
3217 PVM pVM = pVCpu->CTX_SUFF(pVM);
3218 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3219 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3220
3221 /*
3222 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3223 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3224 */
3225 if ( CPUMIsGuestInLongMode(pVCpu)
3226 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3227 {
3228 return true;
3229 }
3230
3231 /*
3232 * If the guest uses PAE and EFER.NXE bit differs, we need to swap as it affects guest paging.
3233 * 64-bit paging implies CR4.PAE as well. See Intel spec. 4.5 "IA32e Paging".
3234 */
3235 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3236 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3237 {
3238 return true;
3239 }
3240
3241 /** @todo Check the latest Intel spec. for any other bits,
3242 * like SMEP/SMAP? */
3243 return false;
3244}
3245
3246
3247/**
3248 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3249 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3250 * controls".
3251 *
3252 * @returns VBox status code.
3253 * @param pVCpu Pointer to the VMCPU.
3254 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3255 * out-of-sync. Make sure to update the required fields
3256 * before using them.
3257 *
3258 * @remarks Requires EFER.
3259 * @remarks No-long-jump zone!!!
3260 */
3261DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3262{
3263 int rc = VINF_SUCCESS;
3264 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3265 {
3266 PVM pVM = pVCpu->CTX_SUFF(pVM);
3267 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3268 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3269
3270 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3271 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3272
3273 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3274 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3275 {
3276 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3277 Log4(("Load: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3278 }
3279 else
3280 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3281
3282 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3283#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3284 if ( HMVMX_IS_64BIT_HOST_MODE()
3285 && pVM->hm.s.vmx.fSupportsVmcsEfer
3286 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3287 {
3288 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3289 Log4(("Load: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3290 }
3291#endif
3292
3293 /*
3294 * The following should -not- be set (since we're not in SMM mode):
3295 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3296 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3297 */
3298
3299 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3300 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3301
3302 if ((val & zap) != val)
3303 {
3304 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3305 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3306 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3307 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3308 }
3309
3310 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3311 AssertRCReturn(rc, rc);
3312
3313 /* Update VCPU with the currently set VM-exit controls. */
3314 pVCpu->hm.s.vmx.u32EntryCtls = val;
3315 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3316 }
3317 return rc;
3318}
3319
3320
3321/**
3322 * Sets up the VM-exit controls in the VMCS.
3323 *
3324 * @returns VBox status code.
3325 * @param pVM Pointer to the VM.
3326 * @param pVCpu Pointer to the VMCPU.
3327 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3328 * out-of-sync. Make sure to update the required fields
3329 * before using them.
3330 *
3331 * @remarks Requires EFER.
3332 */
3333DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3334{
3335 NOREF(pMixedCtx);
3336
3337 int rc = VINF_SUCCESS;
3338 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3339 {
3340 PVM pVM = pVCpu->CTX_SUFF(pVM);
3341 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3342 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3343
3344 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3345 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3346
3347 /*
3348 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3349 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3350 */
3351#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3352 if (HMVMX_IS_64BIT_HOST_MODE())
3353 {
3354 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3355 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3356
3357 /* If the newer VMCS fields for managing EFER exists, use it. */
3358 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3359 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3360 {
3361 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3362 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3363 }
3364 }
3365 else
3366 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3367#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3368 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3369 {
3370 /* The switcher returns to long mode, EFER is managed by the switcher. */
3371 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3372 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3373 }
3374 else
3375 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3376#endif
3377
3378 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3380
3381 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3382 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3383 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3384
3385 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3386 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3387
3388 if ((val & zap) != val)
3389 {
3390 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3391 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3392 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3393 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3394 }
3395
3396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3397 AssertRCReturn(rc, rc);
3398
3399 /* Update VCPU with the currently set VM-exit controls. */
3400 pVCpu->hm.s.vmx.u32ExitCtls = val;
3401 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3402 }
3403 return rc;
3404}
3405
3406
3407/**
3408 * Loads the guest APIC and related state.
3409 *
3410 * @returns VBox status code.
3411 * @param pVM Pointer to the VM.
3412 * @param pVCpu Pointer to the VMCPU.
3413 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3414 * out-of-sync. Make sure to update the required fields
3415 * before using them.
3416 */
3417DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3418{
3419 NOREF(pMixedCtx);
3420
3421 int rc = VINF_SUCCESS;
3422 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3423 {
3424 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3425 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3426 {
3427 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3428
3429 bool fPendingIntr = false;
3430 uint8_t u8Tpr = 0;
3431 uint8_t u8PendingIntr = 0;
3432 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3433 AssertRCReturn(rc, rc);
3434
3435 /*
3436 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3437 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3438 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3439 * the interrupt when we VM-exit for other reasons.
3440 */
3441 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3442 uint32_t u32TprThreshold = 0;
3443 if (fPendingIntr)
3444 {
3445 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3446 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3447 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3448 if (u8PendingPriority <= u8TprPriority)
3449 u32TprThreshold = u8PendingPriority;
3450 else
3451 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3452 }
3453 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3454
3455 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3456 AssertRCReturn(rc, rc);
3457 }
3458
3459 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3460 }
3461 return rc;
3462}
3463
3464
3465/**
3466 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3467 *
3468 * @returns Guest's interruptibility-state.
3469 * @param pVCpu Pointer to the VMCPU.
3470 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3471 * out-of-sync. Make sure to update the required fields
3472 * before using them.
3473 *
3474 * @remarks No-long-jump zone!!!
3475 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3476 */
3477DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3478{
3479 /*
3480 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3481 * inhibit interrupts or clear any existing interrupt-inhibition.
3482 */
3483 uint32_t uIntrState = 0;
3484 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3485 {
3486 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3487 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3488 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3489 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3490 {
3491 /*
3492 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3493 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3494 */
3495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3496 }
3497 else if (pMixedCtx->eflags.Bits.u1IF)
3498 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3499 else
3500 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3501 }
3502 return uIntrState;
3503}
3504
3505
3506/**
3507 * Loads the guest's interruptibility-state into the guest-state area in the
3508 * VMCS.
3509 *
3510 * @returns VBox status code.
3511 * @param pVCpu Pointer to the VMCPU.
3512 * @param uIntrState The interruptibility-state to set.
3513 */
3514static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3515{
3516 NOREF(pVCpu);
3517 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3518 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3519 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3520 AssertRCReturn(rc, rc);
3521 return rc;
3522}
3523
3524
3525/**
3526 * Loads the guest's RIP into the guest-state area in the VMCS.
3527 *
3528 * @returns VBox status code.
3529 * @param pVCpu Pointer to the VMCPU.
3530 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3531 * out-of-sync. Make sure to update the required fields
3532 * before using them.
3533 *
3534 * @remarks No-long-jump zone!!!
3535 */
3536static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3537{
3538 int rc = VINF_SUCCESS;
3539 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3540 {
3541 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3542 AssertRCReturn(rc, rc);
3543
3544 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3545 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3546 }
3547 return rc;
3548}
3549
3550
3551/**
3552 * Loads the guest's RSP into the guest-state area in the VMCS.
3553 *
3554 * @returns VBox status code.
3555 * @param pVCpu Pointer to the VMCPU.
3556 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3557 * out-of-sync. Make sure to update the required fields
3558 * before using them.
3559 *
3560 * @remarks No-long-jump zone!!!
3561 */
3562static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3563{
3564 int rc = VINF_SUCCESS;
3565 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3566 {
3567 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3568 AssertRCReturn(rc, rc);
3569
3570 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3571 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3572 }
3573 return rc;
3574}
3575
3576
3577/**
3578 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3579 *
3580 * @returns VBox status code.
3581 * @param pVCpu Pointer to the VMCPU.
3582 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3583 * out-of-sync. Make sure to update the required fields
3584 * before using them.
3585 *
3586 * @remarks No-long-jump zone!!!
3587 */
3588static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3589{
3590 int rc = VINF_SUCCESS;
3591 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3592 {
3593 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3594 Let us assert it as such and use 32-bit VMWRITE. */
3595 Assert(!(pMixedCtx->rflags.u64 >> 32));
3596 X86EFLAGS Eflags = pMixedCtx->eflags;
3597 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3598 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3599
3600 /*
3601 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3602 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3603 */
3604 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3605 {
3606 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3607 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3608 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3609 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3610 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3611 }
3612
3613 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3614 AssertRCReturn(rc, rc);
3615
3616 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3617 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3618 }
3619 return rc;
3620}
3621
3622
3623/**
3624 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3625 *
3626 * @returns VBox status code.
3627 * @param pVCpu Pointer to the VMCPU.
3628 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3629 * out-of-sync. Make sure to update the required fields
3630 * before using them.
3631 *
3632 * @remarks No-long-jump zone!!!
3633 */
3634DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3635{
3636 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3637 AssertRCReturn(rc, rc);
3638 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3639 AssertRCReturn(rc, rc);
3640 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3641 AssertRCReturn(rc, rc);
3642 return rc;
3643}
3644
3645
3646/**
3647 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3648 * CR0 is partially shared with the host and we have to consider the FPU bits.
3649 *
3650 * @returns VBox status code.
3651 * @param pVM Pointer to the VM.
3652 * @param pVCpu Pointer to the VMCPU.
3653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3654 * out-of-sync. Make sure to update the required fields
3655 * before using them.
3656 *
3657 * @remarks No-long-jump zone!!!
3658 */
3659static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3660{
3661 /*
3662 * Guest CR0.
3663 * Guest FPU.
3664 */
3665 int rc = VINF_SUCCESS;
3666 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3667 {
3668 Assert(!(pMixedCtx->cr0 >> 32));
3669 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3670 PVM pVM = pVCpu->CTX_SUFF(pVM);
3671
3672 /* The guest's view (read access) of its CR0 is unblemished. */
3673 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3674 AssertRCReturn(rc, rc);
3675 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3676
3677 /* Setup VT-x's view of the guest CR0. */
3678 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3679 if (pVM->hm.s.fNestedPaging)
3680 {
3681 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3682 {
3683 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3684 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3685 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3686 }
3687 else
3688 {
3689 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3690 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3691 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3692 }
3693
3694 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3695 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3696 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3697
3698 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3699 AssertRCReturn(rc, rc);
3700 }
3701 else
3702 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3703
3704 /*
3705 * Guest FPU bits.
3706 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3707 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3708 */
3709 u32GuestCR0 |= X86_CR0_NE;
3710 bool fInterceptNM = false;
3711 if (CPUMIsGuestFPUStateActive(pVCpu))
3712 {
3713 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3714 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3715 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3716 }
3717 else
3718 {
3719 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3720 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3721 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3722 }
3723
3724 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3725 bool fInterceptMF = false;
3726 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3727 fInterceptMF = true;
3728
3729 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3730 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3731 {
3732 Assert(PDMVmmDevHeapIsEnabled(pVM));
3733 Assert(pVM->hm.s.vmx.pRealModeTSS);
3734 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3735 fInterceptNM = true;
3736 fInterceptMF = true;
3737 }
3738 else
3739 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3740
3741 if (fInterceptNM)
3742 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3743 else
3744 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3745
3746 if (fInterceptMF)
3747 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3748 else
3749 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3750
3751 /* Additional intercepts for debugging, define these yourself explicitly. */
3752#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3753 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3754 | RT_BIT(X86_XCPT_BP)
3755 | RT_BIT(X86_XCPT_DB)
3756 | RT_BIT(X86_XCPT_DE)
3757 | RT_BIT(X86_XCPT_NM)
3758 | RT_BIT(X86_XCPT_TS)
3759 | RT_BIT(X86_XCPT_UD)
3760 | RT_BIT(X86_XCPT_NP)
3761 | RT_BIT(X86_XCPT_SS)
3762 | RT_BIT(X86_XCPT_GP)
3763 | RT_BIT(X86_XCPT_PF)
3764 | RT_BIT(X86_XCPT_MF)
3765 ;
3766#elif defined(HMVMX_ALWAYS_TRAP_PF)
3767 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3768#endif
3769
3770 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3771
3772 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3773 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3774 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3775 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3776 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3777 else
3778 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3779
3780 u32GuestCR0 |= uSetCR0;
3781 u32GuestCR0 &= uZapCR0;
3782 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3783
3784 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3785 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3786 AssertRCReturn(rc, rc);
3787 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3788 AssertRCReturn(rc, rc);
3789 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3790
3791 /*
3792 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3793 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3794 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3795 */
3796 uint32_t u32CR0Mask = 0;
3797 u32CR0Mask = X86_CR0_PE
3798 | X86_CR0_NE
3799 | X86_CR0_WP
3800 | X86_CR0_PG
3801 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3802 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3803 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3804
3805 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3806 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3807 * and @bugref{6944}. */
3808#if 0
3809 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3810 u32CR0Mask &= ~X86_CR0_PE;
3811#endif
3812 if (pVM->hm.s.fNestedPaging)
3813 u32CR0Mask &= ~X86_CR0_WP;
3814
3815 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3816 if (fInterceptNM)
3817 {
3818 u32CR0Mask |= X86_CR0_TS
3819 | X86_CR0_MP;
3820 }
3821
3822 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3823 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3824 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3825 AssertRCReturn(rc, rc);
3826 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3827
3828 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3829 }
3830 return rc;
3831}
3832
3833
3834/**
3835 * Loads the guest control registers (CR3, CR4) into the guest-state area
3836 * in the VMCS.
3837 *
3838 * @returns VBox status code.
3839 * @param pVM Pointer to the VM.
3840 * @param pVCpu Pointer to the VMCPU.
3841 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3842 * out-of-sync. Make sure to update the required fields
3843 * before using them.
3844 *
3845 * @remarks No-long-jump zone!!!
3846 */
3847static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3848{
3849 int rc = VINF_SUCCESS;
3850 PVM pVM = pVCpu->CTX_SUFF(pVM);
3851
3852 /*
3853 * Guest CR2.
3854 * It's always loaded in the assembler code. Nothing to do here.
3855 */
3856
3857 /*
3858 * Guest CR3.
3859 */
3860 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3861 {
3862 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3863 if (pVM->hm.s.fNestedPaging)
3864 {
3865 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3866
3867 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3868 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3869 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3870 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3871
3872 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3873 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3874 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3875
3876 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3877 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3878 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3879 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3880
3881 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3882 AssertRCReturn(rc, rc);
3883 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3884
3885 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3886 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3887 {
3888 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3889 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3890 {
3891 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3892 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3893 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3894 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3895 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3896 }
3897
3898 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3899 have Unrestricted Execution to handle the guest when it's not using paging. */
3900 GCPhysGuestCR3 = pMixedCtx->cr3;
3901 }
3902 else
3903 {
3904 /*
3905 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3906 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3907 * EPT takes care of translating it to host-physical addresses.
3908 */
3909 RTGCPHYS GCPhys;
3910 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3911 Assert(PDMVmmDevHeapIsEnabled(pVM));
3912
3913 /* We obtain it here every time as the guest could have relocated this PCI region. */
3914 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3915 AssertRCReturn(rc, rc);
3916
3917 GCPhysGuestCR3 = GCPhys;
3918 }
3919
3920 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3921 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3922 }
3923 else
3924 {
3925 /* Non-nested paging case, just use the hypervisor's CR3. */
3926 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3927
3928 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3929 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3930 }
3931 AssertRCReturn(rc, rc);
3932
3933 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3934 }
3935
3936 /*
3937 * Guest CR4.
3938 */
3939 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3940 {
3941 Assert(!(pMixedCtx->cr4 >> 32));
3942 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3943
3944 /* The guest's view of its CR4 is unblemished. */
3945 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3946 AssertRCReturn(rc, rc);
3947 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3948
3949 /* Setup VT-x's view of the guest CR4. */
3950 /*
3951 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3952 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3953 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3954 */
3955 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3956 {
3957 Assert(pVM->hm.s.vmx.pRealModeTSS);
3958 Assert(PDMVmmDevHeapIsEnabled(pVM));
3959 u32GuestCR4 &= ~X86_CR4_VME;
3960 }
3961
3962 if (pVM->hm.s.fNestedPaging)
3963 {
3964 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3965 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3966 {
3967 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3968 u32GuestCR4 |= X86_CR4_PSE;
3969 /* Our identity mapping is a 32-bit page directory. */
3970 u32GuestCR4 &= ~X86_CR4_PAE;
3971 }
3972 /* else use guest CR4.*/
3973 }
3974 else
3975 {
3976 /*
3977 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3978 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3979 */
3980 switch (pVCpu->hm.s.enmShadowMode)
3981 {
3982 case PGMMODE_REAL: /* Real-mode. */
3983 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3984 case PGMMODE_32_BIT: /* 32-bit paging. */
3985 {
3986 u32GuestCR4 &= ~X86_CR4_PAE;
3987 break;
3988 }
3989
3990 case PGMMODE_PAE: /* PAE paging. */
3991 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3992 {
3993 u32GuestCR4 |= X86_CR4_PAE;
3994 break;
3995 }
3996
3997 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3998 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3999#ifdef VBOX_ENABLE_64_BITS_GUESTS
4000 break;
4001#endif
4002 default:
4003 AssertFailed();
4004 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4005 }
4006 }
4007
4008 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4009 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4010 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4011 u32GuestCR4 |= uSetCR4;
4012 u32GuestCR4 &= uZapCR4;
4013
4014 /* Write VT-x's view of the guest CR4 into the VMCS. */
4015 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
4016 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4017 AssertRCReturn(rc, rc);
4018
4019 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4020 uint32_t u32CR4Mask = 0;
4021 u32CR4Mask = X86_CR4_VME
4022 | X86_CR4_PAE
4023 | X86_CR4_PGE
4024 | X86_CR4_PSE
4025 | X86_CR4_VMXE;
4026 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4027 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4028 AssertRCReturn(rc, rc);
4029
4030 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4031 }
4032 return rc;
4033}
4034
4035
4036/**
4037 * Loads the guest debug registers into the guest-state area in the VMCS.
4038 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4039 *
4040 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4041 *
4042 * @returns VBox status code.
4043 * @param pVCpu Pointer to the VMCPU.
4044 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4045 * out-of-sync. Make sure to update the required fields
4046 * before using them.
4047 *
4048 * @remarks No-long-jump zone!!!
4049 */
4050static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4051{
4052 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4053 return VINF_SUCCESS;
4054
4055#ifdef VBOX_STRICT
4056 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4057 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4058 {
4059 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4060 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4061 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4062 }
4063#endif
4064
4065 int rc;
4066 PVM pVM = pVCpu->CTX_SUFF(pVM);
4067 bool fInterceptDB = false;
4068 bool fInterceptMovDRx = false;
4069 if ( pVCpu->hm.s.fSingleInstruction
4070 || DBGFIsStepping(pVCpu))
4071 {
4072 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4073 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4074 {
4075 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4076 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4077 AssertRCReturn(rc, rc);
4078 Assert(fInterceptDB == false);
4079 }
4080 else
4081 {
4082 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4083 pVCpu->hm.s.fClearTrapFlag = true;
4084 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4085 fInterceptDB = true;
4086 }
4087 }
4088
4089 if ( fInterceptDB
4090 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4091 {
4092 /*
4093 * Use the combined guest and host DRx values found in the hypervisor
4094 * register set because the debugger has breakpoints active or someone
4095 * is single stepping on the host side without a monitor trap flag.
4096 *
4097 * Note! DBGF expects a clean DR6 state before executing guest code.
4098 */
4099#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4100 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4101 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4102 {
4103 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4104 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4105 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4106 }
4107 else
4108#endif
4109 if (!CPUMIsHyperDebugStateActive(pVCpu))
4110 {
4111 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4112 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4113 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4114 }
4115
4116 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4117 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4118 AssertRCReturn(rc, rc);
4119
4120 pVCpu->hm.s.fUsingHyperDR7 = true;
4121 fInterceptDB = true;
4122 fInterceptMovDRx = true;
4123 }
4124 else
4125 {
4126 /*
4127 * If the guest has enabled debug registers, we need to load them prior to
4128 * executing guest code so they'll trigger at the right time.
4129 */
4130 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4131 {
4132#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4133 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4134 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4135 {
4136 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4137 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4138 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4139 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4140 }
4141 else
4142#endif
4143 if (!CPUMIsGuestDebugStateActive(pVCpu))
4144 {
4145 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4146 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4147 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4148 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4149 }
4150 Assert(!fInterceptDB);
4151 Assert(!fInterceptMovDRx);
4152 }
4153 /*
4154 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4155 * must intercept #DB in order to maintain a correct DR6 guest value.
4156 */
4157#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4158 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4159 && !CPUMIsGuestDebugStateActive(pVCpu))
4160#else
4161 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4162#endif
4163 {
4164 fInterceptMovDRx = true;
4165 fInterceptDB = true;
4166 }
4167
4168 /* Update guest DR7. */
4169 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4170 AssertRCReturn(rc, rc);
4171
4172 pVCpu->hm.s.fUsingHyperDR7 = false;
4173 }
4174
4175 /*
4176 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4177 */
4178 if (fInterceptDB)
4179 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4180 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4181 {
4182#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4183 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4184#endif
4185 }
4186 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4187 AssertRCReturn(rc, rc);
4188
4189 /*
4190 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4191 */
4192 if (fInterceptMovDRx)
4193 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4194 else
4195 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4196 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4197 AssertRCReturn(rc, rc);
4198
4199 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4200 return VINF_SUCCESS;
4201}
4202
4203
4204#ifdef VBOX_STRICT
4205/**
4206 * Strict function to validate segment registers.
4207 *
4208 * @remarks ASSUMES CR0 is up to date.
4209 */
4210static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4211{
4212 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4213 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4214 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4215 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4216 && ( !CPUMIsGuestInRealModeEx(pCtx)
4217 && !CPUMIsGuestInV86ModeEx(pCtx)))
4218 {
4219 /* Protected mode checks */
4220 /* CS */
4221 Assert(pCtx->cs.Attr.n.u1Present);
4222 Assert(!(pCtx->cs.Attr.u & 0xf00));
4223 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4224 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4225 || !(pCtx->cs.Attr.n.u1Granularity));
4226 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4227 || (pCtx->cs.Attr.n.u1Granularity));
4228 /* CS cannot be loaded with NULL in protected mode. */
4229 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4230 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4231 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4232 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4233 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4234 else
4235 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4236 /* SS */
4237 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4238 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4239 if ( !(pCtx->cr0 & X86_CR0_PE)
4240 || pCtx->cs.Attr.n.u4Type == 3)
4241 {
4242 Assert(!pCtx->ss.Attr.n.u2Dpl);
4243 }
4244 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4245 {
4246 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4247 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4248 Assert(pCtx->ss.Attr.n.u1Present);
4249 Assert(!(pCtx->ss.Attr.u & 0xf00));
4250 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4251 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4252 || !(pCtx->ss.Attr.n.u1Granularity));
4253 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4254 || (pCtx->ss.Attr.n.u1Granularity));
4255 }
4256 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4257 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4258 {
4259 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4260 Assert(pCtx->ds.Attr.n.u1Present);
4261 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4262 Assert(!(pCtx->ds.Attr.u & 0xf00));
4263 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4264 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4265 || !(pCtx->ds.Attr.n.u1Granularity));
4266 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4267 || (pCtx->ds.Attr.n.u1Granularity));
4268 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4269 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4270 }
4271 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4272 {
4273 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4274 Assert(pCtx->es.Attr.n.u1Present);
4275 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4276 Assert(!(pCtx->es.Attr.u & 0xf00));
4277 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4278 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4279 || !(pCtx->es.Attr.n.u1Granularity));
4280 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4281 || (pCtx->es.Attr.n.u1Granularity));
4282 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4283 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4284 }
4285 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4286 {
4287 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4288 Assert(pCtx->fs.Attr.n.u1Present);
4289 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4290 Assert(!(pCtx->fs.Attr.u & 0xf00));
4291 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4292 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4293 || !(pCtx->fs.Attr.n.u1Granularity));
4294 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4295 || (pCtx->fs.Attr.n.u1Granularity));
4296 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4297 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4298 }
4299 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4300 {
4301 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4302 Assert(pCtx->gs.Attr.n.u1Present);
4303 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4304 Assert(!(pCtx->gs.Attr.u & 0xf00));
4305 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4306 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4307 || !(pCtx->gs.Attr.n.u1Granularity));
4308 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4309 || (pCtx->gs.Attr.n.u1Granularity));
4310 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4311 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4312 }
4313 /* 64-bit capable CPUs. */
4314# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4315 if (HMVMX_IS_64BIT_HOST_MODE())
4316 {
4317 Assert(!(pCtx->cs.u64Base >> 32));
4318 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4319 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4320 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4321 }
4322# endif
4323 }
4324 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4325 || ( CPUMIsGuestInRealModeEx(pCtx)
4326 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4327 {
4328 /* Real and v86 mode checks. */
4329 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4330 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4331 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4332 {
4333 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4334 }
4335 else
4336 {
4337 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4338 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4339 }
4340
4341 /* CS */
4342 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4343 Assert(pCtx->cs.u32Limit == 0xffff);
4344 Assert(u32CSAttr == 0xf3);
4345 /* SS */
4346 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4347 Assert(pCtx->ss.u32Limit == 0xffff);
4348 Assert(u32SSAttr == 0xf3);
4349 /* DS */
4350 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4351 Assert(pCtx->ds.u32Limit == 0xffff);
4352 Assert(u32DSAttr == 0xf3);
4353 /* ES */
4354 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4355 Assert(pCtx->es.u32Limit == 0xffff);
4356 Assert(u32ESAttr == 0xf3);
4357 /* FS */
4358 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4359 Assert(pCtx->fs.u32Limit == 0xffff);
4360 Assert(u32FSAttr == 0xf3);
4361 /* GS */
4362 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4363 Assert(pCtx->gs.u32Limit == 0xffff);
4364 Assert(u32GSAttr == 0xf3);
4365 /* 64-bit capable CPUs. */
4366# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4367 if (HMVMX_IS_64BIT_HOST_MODE())
4368 {
4369 Assert(!(pCtx->cs.u64Base >> 32));
4370 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4371 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4372 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4373 }
4374# endif
4375 }
4376}
4377#endif /* VBOX_STRICT */
4378
4379
4380/**
4381 * Writes a guest segment register into the guest-state area in the VMCS.
4382 *
4383 * @returns VBox status code.
4384 * @param pVCpu Pointer to the VMCPU.
4385 * @param idxSel Index of the selector in the VMCS.
4386 * @param idxLimit Index of the segment limit in the VMCS.
4387 * @param idxBase Index of the segment base in the VMCS.
4388 * @param idxAccess Index of the access rights of the segment in the VMCS.
4389 * @param pSelReg Pointer to the segment selector.
4390 *
4391 * @remarks No-long-jump zone!!!
4392 */
4393static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4394 uint32_t idxAccess, PCPUMSELREG pSelReg)
4395{
4396 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4397 AssertRCReturn(rc, rc);
4398 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4399 AssertRCReturn(rc, rc);
4400 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4401 AssertRCReturn(rc, rc);
4402
4403 uint32_t u32Access = pSelReg->Attr.u;
4404 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4405 {
4406 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4407 u32Access = 0xf3;
4408 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4409 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4410 }
4411 else
4412 {
4413 /*
4414 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4415 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4416 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4417 * loaded in protected-mode have their attribute as 0.
4418 */
4419 if (!u32Access)
4420 u32Access = X86DESCATTR_UNUSABLE;
4421 }
4422
4423 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4424 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4425 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4426
4427 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4428 AssertRCReturn(rc, rc);
4429 return rc;
4430}
4431
4432
4433/**
4434 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4435 * into the guest-state area in the VMCS.
4436 *
4437 * @returns VBox status code.
4438 * @param pVM Pointer to the VM.
4439 * @param pVCPU Pointer to the VMCPU.
4440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4441 * out-of-sync. Make sure to update the required fields
4442 * before using them.
4443 *
4444 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4445 * @remarks No-long-jump zone!!!
4446 */
4447static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4448{
4449 int rc = VERR_INTERNAL_ERROR_5;
4450 PVM pVM = pVCpu->CTX_SUFF(pVM);
4451
4452 /*
4453 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4454 */
4455 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4456 {
4457 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4458 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4459 {
4460 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4461 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4462 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4463 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4464 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4465 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4466 }
4467
4468#ifdef VBOX_WITH_REM
4469 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4470 {
4471 Assert(pVM->hm.s.vmx.pRealModeTSS);
4472 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4473 if ( pVCpu->hm.s.vmx.fWasInRealMode
4474 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4475 {
4476 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4477 in real-mode (e.g. OpenBSD 4.0) */
4478 REMFlushTBs(pVM);
4479 Log4(("Load: Switch to protected mode detected!\n"));
4480 pVCpu->hm.s.vmx.fWasInRealMode = false;
4481 }
4482 }
4483#endif
4484 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4485 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4486 AssertRCReturn(rc, rc);
4487 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4488 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4489 AssertRCReturn(rc, rc);
4490 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4491 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4492 AssertRCReturn(rc, rc);
4493 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4494 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4495 AssertRCReturn(rc, rc);
4496 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4497 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4498 AssertRCReturn(rc, rc);
4499 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4500 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4501 AssertRCReturn(rc, rc);
4502
4503#ifdef VBOX_STRICT
4504 /* Validate. */
4505 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4506#endif
4507
4508 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4509 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4510 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4511 }
4512
4513 /*
4514 * Guest TR.
4515 */
4516 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4517 {
4518 /*
4519 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4520 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4521 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4522 */
4523 uint16_t u16Sel = 0;
4524 uint32_t u32Limit = 0;
4525 uint64_t u64Base = 0;
4526 uint32_t u32AccessRights = 0;
4527
4528 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4529 {
4530 u16Sel = pMixedCtx->tr.Sel;
4531 u32Limit = pMixedCtx->tr.u32Limit;
4532 u64Base = pMixedCtx->tr.u64Base;
4533 u32AccessRights = pMixedCtx->tr.Attr.u;
4534 }
4535 else
4536 {
4537 Assert(pVM->hm.s.vmx.pRealModeTSS);
4538 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4539
4540 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4541 RTGCPHYS GCPhys;
4542 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4543 AssertRCReturn(rc, rc);
4544
4545 X86DESCATTR DescAttr;
4546 DescAttr.u = 0;
4547 DescAttr.n.u1Present = 1;
4548 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4549
4550 u16Sel = 0;
4551 u32Limit = HM_VTX_TSS_SIZE;
4552 u64Base = GCPhys; /* in real-mode phys = virt. */
4553 u32AccessRights = DescAttr.u;
4554 }
4555
4556 /* Validate. */
4557 Assert(!(u16Sel & RT_BIT(2)));
4558 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4559 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4560 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4561 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4562 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4563 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4564 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4565 Assert( (u32Limit & 0xfff) == 0xfff
4566 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4567 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4568 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4569
4570 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4571 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4572 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4573 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4574
4575 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4576 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4577 }
4578
4579 /*
4580 * Guest GDTR.
4581 */
4582 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4583 {
4584 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4585 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4586
4587 /* Validate. */
4588 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4589
4590 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4591 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4592 }
4593
4594 /*
4595 * Guest LDTR.
4596 */
4597 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4598 {
4599 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4600 uint32_t u32Access = 0;
4601 if (!pMixedCtx->ldtr.Attr.u)
4602 u32Access = X86DESCATTR_UNUSABLE;
4603 else
4604 u32Access = pMixedCtx->ldtr.Attr.u;
4605
4606 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4607 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4608 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4610
4611 /* Validate. */
4612 if (!(u32Access & X86DESCATTR_UNUSABLE))
4613 {
4614 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4615 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4616 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4617 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4618 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4619 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4620 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4621 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4622 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4623 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4624 }
4625
4626 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4627 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4628 }
4629
4630 /*
4631 * Guest IDTR.
4632 */
4633 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4634 {
4635 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4636 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4637
4638 /* Validate. */
4639 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4640
4641 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4642 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4643 }
4644
4645 return VINF_SUCCESS;
4646}
4647
4648
4649/**
4650 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4651 * areas. These MSRs will automatically be loaded to the host CPU on every
4652 * successful VM entry and stored from the host CPU on every successful VM-exit.
4653 *
4654 * This also creates/updates MSR slots for the host MSRs. The actual host
4655 * MSR values are -not- updated here for performance reasons. See
4656 * hmR0VmxSaveHostMsrs().
4657 *
4658 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4659 *
4660 * @returns VBox status code.
4661 * @param pVCpu Pointer to the VMCPU.
4662 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4663 * out-of-sync. Make sure to update the required fields
4664 * before using them.
4665 *
4666 * @remarks No-long-jump zone!!!
4667 */
4668static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4669{
4670 AssertPtr(pVCpu);
4671 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4672
4673 /*
4674 * MSRs that we use the auto-load/store MSR area in the VMCS.
4675 */
4676 PVM pVM = pVCpu->CTX_SUFF(pVM);
4677 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4678 {
4679 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4680#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4681 if (pVM->hm.s.fAllow64BitGuests)
4682 {
4683 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4684 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4685 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4686 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4687# ifdef DEBUG
4688 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4689 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4690 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4691# endif
4692 }
4693#endif
4694 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4695 }
4696
4697 /*
4698 * Guest Sysenter MSRs.
4699 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4700 * VM-exits on WRMSRs for these MSRs.
4701 */
4702 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4703 {
4704 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4705 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4706 }
4707
4708 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4709 {
4710 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4711 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4712 }
4713
4714 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4715 {
4716 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4717 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4718 }
4719
4720 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4721 {
4722#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4723 if (HMVMX_IS_64BIT_HOST_MODE())
4724 {
4725 /*
4726 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4727 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4728 */
4729 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4730 {
4731 /* Not strictly necessary to check hmR0VmxShouldSwapEferMsr() here, but it avoids
4732 one VM-write when we're a nested guest. */
4733 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4734 {
4735 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4736 AssertRCReturn(rc,rc);
4737 Log4(("Load: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pMixedCtx->msrEFER));
4738 }
4739 }
4740 else
4741 {
4742 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4743 {
4744 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4745 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4746 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4747 Log4(("Load: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4748 pVCpu->hm.s.vmx.cMsrs));
4749 }
4750 else
4751 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4752 }
4753 }
4754#endif
4755 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4756 }
4757
4758 return VINF_SUCCESS;
4759}
4760
4761
4762/**
4763 * Loads the guest activity state into the guest-state area in the VMCS.
4764 *
4765 * @returns VBox status code.
4766 * @param pVCpu Pointer to the VMCPU.
4767 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4768 * out-of-sync. Make sure to update the required fields
4769 * before using them.
4770 *
4771 * @remarks No-long-jump zone!!!
4772 */
4773static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4774{
4775 NOREF(pCtx);
4776 /** @todo See if we can make use of other states, e.g.
4777 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4778 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4779 {
4780 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4781 AssertRCReturn(rc, rc);
4782
4783 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4784 }
4785 return VINF_SUCCESS;
4786}
4787
4788
4789/**
4790 * Sets up the appropriate function to run guest code.
4791 *
4792 * @returns VBox status code.
4793 * @param pVCpu Pointer to the VMCPU.
4794 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4795 * out-of-sync. Make sure to update the required fields
4796 * before using them.
4797 *
4798 * @remarks No-long-jump zone!!!
4799 */
4800static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4801{
4802 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4803 {
4804#ifndef VBOX_ENABLE_64_BITS_GUESTS
4805 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4806#endif
4807 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4808#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4809 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4810 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4811 {
4812 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4813 {
4814 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4815 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4816 | HM_CHANGED_VMX_EXIT_CTLS
4817 | HM_CHANGED_VMX_ENTRY_CTLS
4818 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4819 }
4820 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4821 }
4822#else
4823 /* 64-bit host or hybrid host. */
4824 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4825#endif
4826 }
4827 else
4828 {
4829 /* Guest is not in long mode, use the 32-bit handler. */
4830#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4831 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4832 {
4833 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4834 {
4835 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4836 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4837 | HM_CHANGED_VMX_EXIT_CTLS
4838 | HM_CHANGED_VMX_ENTRY_CTLS
4839 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4840 }
4841 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4842 }
4843#else
4844 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4845#endif
4846 }
4847 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4848 return VINF_SUCCESS;
4849}
4850
4851
4852/**
4853 * Wrapper for running the guest code in VT-x.
4854 *
4855 * @returns VBox strict status code.
4856 * @param pVM Pointer to the VM.
4857 * @param pVCpu Pointer to the VMCPU.
4858 * @param pCtx Pointer to the guest-CPU context.
4859 *
4860 * @remarks No-long-jump zone!!!
4861 */
4862DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4863{
4864 /*
4865 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4866 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4867 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4868 */
4869 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4870 /** @todo Add stats for resume vs launch. */
4871#ifdef VBOX_WITH_KERNEL_USING_XMM
4872 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4873#else
4874 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4875#endif
4876}
4877
4878
4879/**
4880 * Reports world-switch error and dumps some useful debug info.
4881 *
4882 * @param pVM Pointer to the VM.
4883 * @param pVCpu Pointer to the VMCPU.
4884 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4885 * @param pCtx Pointer to the guest-CPU context.
4886 * @param pVmxTransient Pointer to the VMX transient structure (only
4887 * exitReason updated).
4888 */
4889static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4890{
4891 Assert(pVM);
4892 Assert(pVCpu);
4893 Assert(pCtx);
4894 Assert(pVmxTransient);
4895 HMVMX_ASSERT_PREEMPT_SAFE();
4896
4897 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4898 switch (rcVMRun)
4899 {
4900 case VERR_VMX_INVALID_VMXON_PTR:
4901 AssertFailed();
4902 break;
4903 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4904 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4905 {
4906 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4907 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4908 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4909 AssertRC(rc);
4910
4911 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4912 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4913 Cannot do it here as we may have been long preempted. */
4914
4915#ifdef VBOX_STRICT
4916 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4917 pVmxTransient->uExitReason));
4918 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4919 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4920 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4921 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4922 else
4923 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4924 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4925 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4926
4927 /* VMX control bits. */
4928 uint32_t u32Val;
4929 uint64_t u64Val;
4930 HMVMXHCUINTREG uHCReg;
4931 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4932 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4933 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4934 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4935 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4936 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4937 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4938 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4939 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4940 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4941 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4942 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4943 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4944 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4945 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4946 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4947 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4948 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4949 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4950 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4951 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4952 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4953 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4954 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4955 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4956 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4957 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4958 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4959 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4960 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4961 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4962 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4963 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4964 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4965 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4966 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4967 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4968 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4969 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4970 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4971 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4972 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4973
4974 /* Guest bits. */
4975 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4976 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4977 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4978 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4979 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4980 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4983
4984 /* Host bits. */
4985 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4986 Log4(("Host CR0 %#RHr\n", uHCReg));
4987 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4988 Log4(("Host CR3 %#RHr\n", uHCReg));
4989 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4990 Log4(("Host CR4 %#RHr\n", uHCReg));
4991
4992 RTGDTR HostGdtr;
4993 PCX86DESCHC pDesc;
4994 ASMGetGDTR(&HostGdtr);
4995 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4996 Log4(("Host CS %#08x\n", u32Val));
4997 if (u32Val < HostGdtr.cbGdt)
4998 {
4999 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5000 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5001 }
5002
5003 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5004 Log4(("Host DS %#08x\n", u32Val));
5005 if (u32Val < HostGdtr.cbGdt)
5006 {
5007 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5008 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5009 }
5010
5011 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5012 Log4(("Host ES %#08x\n", u32Val));
5013 if (u32Val < HostGdtr.cbGdt)
5014 {
5015 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5016 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5017 }
5018
5019 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5020 Log4(("Host FS %#08x\n", u32Val));
5021 if (u32Val < HostGdtr.cbGdt)
5022 {
5023 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5024 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5025 }
5026
5027 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5028 Log4(("Host GS %#08x\n", u32Val));
5029 if (u32Val < HostGdtr.cbGdt)
5030 {
5031 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5032 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5033 }
5034
5035 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5036 Log4(("Host SS %#08x\n", u32Val));
5037 if (u32Val < HostGdtr.cbGdt)
5038 {
5039 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5040 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5041 }
5042
5043 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5044 Log4(("Host TR %#08x\n", u32Val));
5045 if (u32Val < HostGdtr.cbGdt)
5046 {
5047 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5048 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5049 }
5050
5051 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5052 Log4(("Host TR Base %#RHv\n", uHCReg));
5053 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5054 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5055 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5056 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5057 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5058 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5059 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5060 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5061 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5062 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5063 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5064 Log4(("Host RSP %#RHv\n", uHCReg));
5065 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5066 Log4(("Host RIP %#RHv\n", uHCReg));
5067# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5068 if (HMVMX_IS_64BIT_HOST_MODE())
5069 {
5070 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5071 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5072 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5073 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5074 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5075 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5076 }
5077# endif
5078#endif /* VBOX_STRICT */
5079 break;
5080 }
5081
5082 default:
5083 /* Impossible */
5084 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5085 break;
5086 }
5087 NOREF(pVM); NOREF(pCtx);
5088}
5089
5090
5091#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5092#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5093# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5094#endif
5095#ifdef VBOX_STRICT
5096static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5097{
5098 switch (idxField)
5099 {
5100 case VMX_VMCS_GUEST_RIP:
5101 case VMX_VMCS_GUEST_RSP:
5102 case VMX_VMCS_GUEST_SYSENTER_EIP:
5103 case VMX_VMCS_GUEST_SYSENTER_ESP:
5104 case VMX_VMCS_GUEST_GDTR_BASE:
5105 case VMX_VMCS_GUEST_IDTR_BASE:
5106 case VMX_VMCS_GUEST_CS_BASE:
5107 case VMX_VMCS_GUEST_DS_BASE:
5108 case VMX_VMCS_GUEST_ES_BASE:
5109 case VMX_VMCS_GUEST_FS_BASE:
5110 case VMX_VMCS_GUEST_GS_BASE:
5111 case VMX_VMCS_GUEST_SS_BASE:
5112 case VMX_VMCS_GUEST_LDTR_BASE:
5113 case VMX_VMCS_GUEST_TR_BASE:
5114 case VMX_VMCS_GUEST_CR3:
5115 return true;
5116 }
5117 return false;
5118}
5119
5120static bool hmR0VmxIsValidReadField(uint32_t idxField)
5121{
5122 switch (idxField)
5123 {
5124 /* Read-only fields. */
5125 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5126 return true;
5127 }
5128 /* Remaining readable fields should also be writable. */
5129 return hmR0VmxIsValidWriteField(idxField);
5130}
5131#endif /* VBOX_STRICT */
5132
5133
5134/**
5135 * Executes the specified handler in 64-bit mode.
5136 *
5137 * @returns VBox status code.
5138 * @param pVM Pointer to the VM.
5139 * @param pVCpu Pointer to the VMCPU.
5140 * @param pCtx Pointer to the guest CPU context.
5141 * @param enmOp The operation to perform.
5142 * @param cbParam Number of parameters.
5143 * @param paParam Array of 32-bit parameters.
5144 */
5145VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5146 uint32_t *paParam)
5147{
5148 int rc, rc2;
5149 PHMGLOBALCPUINFO pCpu;
5150 RTHCPHYS HCPhysCpuPage;
5151 RTCCUINTREG uOldEflags;
5152
5153 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5154 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5155 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5156 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5157
5158#ifdef VBOX_STRICT
5159 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5160 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5161
5162 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5163 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5164#endif
5165
5166 /* Disable interrupts. */
5167 uOldEflags = ASMIntDisableFlags();
5168
5169#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5170 RTCPUID idHostCpu = RTMpCpuId();
5171 CPUMR0SetLApic(pVCpu, idHostCpu);
5172#endif
5173
5174 pCpu = HMR0GetCurrentCpu();
5175 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5176
5177 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5178 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5179
5180 /* Leave VMX Root Mode. */
5181 VMXDisable();
5182
5183 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5184
5185 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5186 CPUMSetHyperEIP(pVCpu, enmOp);
5187 for (int i = (int)cbParam - 1; i >= 0; i--)
5188 CPUMPushHyper(pVCpu, paParam[i]);
5189
5190 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5191
5192 /* Call the switcher. */
5193 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5194 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5195
5196 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5197 /* Make sure the VMX instructions don't cause #UD faults. */
5198 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5199
5200 /* Re-enter VMX Root Mode */
5201 rc2 = VMXEnable(HCPhysCpuPage);
5202 if (RT_FAILURE(rc2))
5203 {
5204 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5205 ASMSetFlags(uOldEflags);
5206 return rc2;
5207 }
5208
5209 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5210 AssertRC(rc2);
5211 Assert(!(ASMGetFlags() & X86_EFL_IF));
5212 ASMSetFlags(uOldEflags);
5213 return rc;
5214}
5215
5216
5217/**
5218 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5219 * supporting 64-bit guests.
5220 *
5221 * @returns VBox status code.
5222 * @param fResume Whether to VMLAUNCH or VMRESUME.
5223 * @param pCtx Pointer to the guest-CPU context.
5224 * @param pCache Pointer to the VMCS cache.
5225 * @param pVM Pointer to the VM.
5226 * @param pVCpu Pointer to the VMCPU.
5227 */
5228DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5229{
5230 uint32_t aParam[6];
5231 PHMGLOBALCPUINFO pCpu = NULL;
5232 RTHCPHYS HCPhysCpuPage = 0;
5233 int rc = VERR_INTERNAL_ERROR_5;
5234
5235 pCpu = HMR0GetCurrentCpu();
5236 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5237
5238#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5239 pCache->uPos = 1;
5240 pCache->interPD = PGMGetInterPaeCR3(pVM);
5241 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5242#endif
5243
5244#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5245 pCache->TestIn.HCPhysCpuPage = 0;
5246 pCache->TestIn.HCPhysVmcs = 0;
5247 pCache->TestIn.pCache = 0;
5248 pCache->TestOut.HCPhysVmcs = 0;
5249 pCache->TestOut.pCache = 0;
5250 pCache->TestOut.pCtx = 0;
5251 pCache->TestOut.eflags = 0;
5252#endif
5253
5254 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5255 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5256 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5257 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5258 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5259 aParam[5] = 0;
5260
5261#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5262 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5263 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5264#endif
5265 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5266
5267#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5268 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5269 Assert(pCtx->dr[4] == 10);
5270 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5271#endif
5272
5273#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5274 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5275 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5276 pVCpu->hm.s.vmx.HCPhysVmcs));
5277 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5278 pCache->TestOut.HCPhysVmcs));
5279 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5280 pCache->TestOut.pCache));
5281 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5282 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5283 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5284 pCache->TestOut.pCtx));
5285 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5286#endif
5287 return rc;
5288}
5289
5290
5291/**
5292 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5293 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5294 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5295 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5296 *
5297 * @returns VBox status code.
5298 * @param pVM Pointer to the VM.
5299 * @param pVCpu Pointer to the VMCPU.
5300 */
5301static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5302{
5303#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5304{ \
5305 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5306 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5307 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5308 ++cReadFields; \
5309}
5310
5311 AssertPtr(pVM);
5312 AssertPtr(pVCpu);
5313 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5314 uint32_t cReadFields = 0;
5315
5316 /*
5317 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5318 * and serve to indicate exceptions to the rules.
5319 */
5320
5321 /* Guest-natural selector base fields. */
5322#if 0
5323 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5324 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5325 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5326#endif
5327 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5328 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5329 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5330 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5331 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5332 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5333 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5334 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5335 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5336 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5337 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5338 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5339#if 0
5340 /* Unused natural width guest-state fields. */
5341 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5342 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5343#endif
5344 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5345 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5346
5347 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5348#if 0
5349 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5352 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5358#endif
5359
5360 /* Natural width guest-state fields. */
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5362#if 0
5363 /* Currently unused field. */
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5365#endif
5366
5367 if (pVM->hm.s.fNestedPaging)
5368 {
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5370 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5371 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5372 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5373 }
5374 else
5375 {
5376 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5377 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5378 }
5379
5380#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5381 return VINF_SUCCESS;
5382}
5383
5384
5385/**
5386 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5387 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5388 * darwin, running 64-bit guests).
5389 *
5390 * @returns VBox status code.
5391 * @param pVCpu Pointer to the VMCPU.
5392 * @param idxField The VMCS field encoding.
5393 * @param u64Val 16, 32 or 64-bit value.
5394 */
5395VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5396{
5397 int rc;
5398 switch (idxField)
5399 {
5400 /*
5401 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5402 */
5403 /* 64-bit Control fields. */
5404 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5405 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5406 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5407 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5408 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5409 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5410 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5411 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5412 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5413 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5414 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5415 case VMX_VMCS64_CTRL_EPTP_FULL:
5416 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5417 /* 64-bit Guest-state fields. */
5418 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5419 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5420 case VMX_VMCS64_GUEST_PAT_FULL:
5421 case VMX_VMCS64_GUEST_EFER_FULL:
5422 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5423 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5424 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5425 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5426 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5427 /* 64-bit Host-state fields. */
5428 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5429 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5430 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5431 {
5432 rc = VMXWriteVmcs32(idxField, u64Val);
5433 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5434 break;
5435 }
5436
5437 /*
5438 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5439 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5440 */
5441 /* Natural-width Guest-state fields. */
5442 case VMX_VMCS_GUEST_CR3:
5443 case VMX_VMCS_GUEST_ES_BASE:
5444 case VMX_VMCS_GUEST_CS_BASE:
5445 case VMX_VMCS_GUEST_SS_BASE:
5446 case VMX_VMCS_GUEST_DS_BASE:
5447 case VMX_VMCS_GUEST_FS_BASE:
5448 case VMX_VMCS_GUEST_GS_BASE:
5449 case VMX_VMCS_GUEST_LDTR_BASE:
5450 case VMX_VMCS_GUEST_TR_BASE:
5451 case VMX_VMCS_GUEST_GDTR_BASE:
5452 case VMX_VMCS_GUEST_IDTR_BASE:
5453 case VMX_VMCS_GUEST_RSP:
5454 case VMX_VMCS_GUEST_RIP:
5455 case VMX_VMCS_GUEST_SYSENTER_ESP:
5456 case VMX_VMCS_GUEST_SYSENTER_EIP:
5457 {
5458 if (!(u64Val >> 32))
5459 {
5460 /* If this field is 64-bit, VT-x will zero out the top bits. */
5461 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5462 }
5463 else
5464 {
5465 /* Assert that only the 32->64 switcher case should ever come here. */
5466 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5467 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5468 }
5469 break;
5470 }
5471
5472 default:
5473 {
5474 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5475 rc = VERR_INVALID_PARAMETER;
5476 break;
5477 }
5478 }
5479 AssertRCReturn(rc, rc);
5480 return rc;
5481}
5482
5483
5484/**
5485 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5486 * hosts (except darwin) for 64-bit guests.
5487 *
5488 * @param pVCpu Pointer to the VMCPU.
5489 * @param idxField The VMCS field encoding.
5490 * @param u64Val 16, 32 or 64-bit value.
5491 */
5492VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5493{
5494 AssertPtr(pVCpu);
5495 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5496
5497 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5498 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5499
5500 /* Make sure there are no duplicates. */
5501 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5502 {
5503 if (pCache->Write.aField[i] == idxField)
5504 {
5505 pCache->Write.aFieldVal[i] = u64Val;
5506 return VINF_SUCCESS;
5507 }
5508 }
5509
5510 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5511 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5512 pCache->Write.cValidEntries++;
5513 return VINF_SUCCESS;
5514}
5515
5516/* Enable later when the assembly code uses these as callbacks. */
5517#if 0
5518/*
5519 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5520 *
5521 * @param pVCpu Pointer to the VMCPU.
5522 * @param pCache Pointer to the VMCS cache.
5523 *
5524 * @remarks No-long-jump zone!!!
5525 */
5526VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5527{
5528 AssertPtr(pCache);
5529 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5530 {
5531 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5532 AssertRC(rc);
5533 }
5534 pCache->Write.cValidEntries = 0;
5535}
5536
5537
5538/**
5539 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5540 *
5541 * @param pVCpu Pointer to the VMCPU.
5542 * @param pCache Pointer to the VMCS cache.
5543 *
5544 * @remarks No-long-jump zone!!!
5545 */
5546VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5547{
5548 AssertPtr(pCache);
5549 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5550 {
5551 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5552 AssertRC(rc);
5553 }
5554}
5555#endif
5556#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5557
5558
5559/**
5560 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5561 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5562 * timer.
5563 *
5564 * @returns VBox status code.
5565 * @param pVCpu Pointer to the VMCPU.
5566 *
5567 * @remarks No-long-jump zone!!!
5568 */
5569static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5570{
5571 int rc = VERR_INTERNAL_ERROR_5;
5572 bool fOffsettedTsc = false;
5573 PVM pVM = pVCpu->CTX_SUFF(pVM);
5574 if (pVM->hm.s.vmx.fUsePreemptTimer)
5575 {
5576 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5577
5578 /* Make sure the returned values have sane upper and lower boundaries. */
5579 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5580 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5581 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5582 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5583
5584 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5585 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5586 }
5587 else
5588 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5589
5590 if (fOffsettedTsc)
5591 {
5592 uint64_t u64CurTSC = ASMReadTSC();
5593 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5594 {
5595 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5596 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5597
5598 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5599 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5600 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5601 }
5602 else
5603 {
5604 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5605 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5606 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5607 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5608 }
5609 }
5610 else
5611 {
5612 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5613 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5614 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5615 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5616 }
5617}
5618
5619
5620/**
5621 * Determines if an exception is a contributory exception. Contributory
5622 * exceptions are ones which can cause double-faults. Page-fault is
5623 * intentionally not included here as it's a conditional contributory exception.
5624 *
5625 * @returns true if the exception is contributory, false otherwise.
5626 * @param uVector The exception vector.
5627 */
5628DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5629{
5630 switch (uVector)
5631 {
5632 case X86_XCPT_GP:
5633 case X86_XCPT_SS:
5634 case X86_XCPT_NP:
5635 case X86_XCPT_TS:
5636 case X86_XCPT_DE:
5637 return true;
5638 default:
5639 break;
5640 }
5641 return false;
5642}
5643
5644
5645/**
5646 * Sets an event as a pending event to be injected into the guest.
5647 *
5648 * @param pVCpu Pointer to the VMCPU.
5649 * @param u32IntInfo The VM-entry interruption-information field.
5650 * @param cbInstr The VM-entry instruction length in bytes (for software
5651 * interrupts, exceptions and privileged software
5652 * exceptions).
5653 * @param u32ErrCode The VM-entry exception error code.
5654 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5655 * page-fault.
5656 *
5657 * @remarks Statistics counter assumes this is a guest event being injected or
5658 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5659 * always incremented.
5660 */
5661DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5662 RTGCUINTPTR GCPtrFaultAddress)
5663{
5664 Assert(!pVCpu->hm.s.Event.fPending);
5665 pVCpu->hm.s.Event.fPending = true;
5666 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5667 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5668 pVCpu->hm.s.Event.cbInstr = cbInstr;
5669 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5670
5671 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5672}
5673
5674
5675/**
5676 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5677 *
5678 * @param pVCpu Pointer to the VMCPU.
5679 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5680 * out-of-sync. Make sure to update the required fields
5681 * before using them.
5682 */
5683DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5684{
5685 NOREF(pMixedCtx);
5686 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5687 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5688 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5689 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5690}
5691
5692
5693/**
5694 * Handle a condition that occurred while delivering an event through the guest
5695 * IDT.
5696 *
5697 * @returns VBox status code (informational error codes included).
5698 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5699 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5700 * continue execution of the guest which will delivery the #DF.
5701 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5702 *
5703 * @param pVCpu Pointer to the VMCPU.
5704 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5705 * out-of-sync. Make sure to update the required fields
5706 * before using them.
5707 * @param pVmxTransient Pointer to the VMX transient structure.
5708 *
5709 * @remarks No-long-jump zone!!!
5710 */
5711static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5712{
5713 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5714 AssertRCReturn(rc, rc);
5715 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5716 {
5717 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5718 AssertRCReturn(rc, rc);
5719
5720 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5721 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5722 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5723
5724 typedef enum
5725 {
5726 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5727 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5728 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5729 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5730 } VMXREFLECTXCPT;
5731
5732 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5733 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5734 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5735 {
5736 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5737 {
5738 enmReflect = VMXREFLECTXCPT_XCPT;
5739#ifdef VBOX_STRICT
5740 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5741 && uExitVector == X86_XCPT_PF)
5742 {
5743 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5744 }
5745#endif
5746 if ( uExitVector == X86_XCPT_PF
5747 && uIdtVector == X86_XCPT_PF)
5748 {
5749 pVmxTransient->fVectoringPF = true;
5750 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5751 }
5752 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5753 && hmR0VmxIsContributoryXcpt(uExitVector)
5754 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5755 || uIdtVector == X86_XCPT_PF))
5756 {
5757 enmReflect = VMXREFLECTXCPT_DF;
5758 }
5759 else if (uIdtVector == X86_XCPT_DF)
5760 enmReflect = VMXREFLECTXCPT_TF;
5761 }
5762 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5763 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5764 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5765 {
5766 /*
5767 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5768 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5769 */
5770 enmReflect = VMXREFLECTXCPT_XCPT;
5771 }
5772 }
5773 else
5774 {
5775 /*
5776 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5777 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5778 * original exception to the guest after handling the VM-exit.
5779 */
5780 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5781 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5782 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5783 {
5784 enmReflect = VMXREFLECTXCPT_XCPT;
5785 }
5786 }
5787
5788 switch (enmReflect)
5789 {
5790 case VMXREFLECTXCPT_XCPT:
5791 {
5792 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5793 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5794 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5795
5796 uint32_t u32ErrCode = 0;
5797 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5798 {
5799 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5800 AssertRCReturn(rc, rc);
5801 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5802 }
5803
5804 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5805 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5806 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5807 rc = VINF_SUCCESS;
5808 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5809 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5810
5811 break;
5812 }
5813
5814 case VMXREFLECTXCPT_DF:
5815 {
5816 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5817 rc = VINF_HM_DOUBLE_FAULT;
5818 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5819 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5820
5821 break;
5822 }
5823
5824 case VMXREFLECTXCPT_TF:
5825 {
5826 rc = VINF_EM_RESET;
5827 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5828 uExitVector));
5829 break;
5830 }
5831
5832 default:
5833 Assert(rc == VINF_SUCCESS);
5834 break;
5835 }
5836 }
5837 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5838 return rc;
5839}
5840
5841
5842/**
5843 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5844 *
5845 * @returns VBox status code.
5846 * @param pVCpu Pointer to the VMCPU.
5847 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5848 * out-of-sync. Make sure to update the required fields
5849 * before using them.
5850 *
5851 * @remarks No-long-jump zone!!!
5852 */
5853static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5854{
5855 NOREF(pMixedCtx);
5856
5857 /*
5858 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5859 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5860 */
5861 VMMRZCallRing3Disable(pVCpu);
5862 HM_DISABLE_PREEMPT_IF_NEEDED();
5863
5864 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5865 {
5866 uint32_t uVal = 0;
5867 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5868 AssertRCReturn(rc, rc);
5869
5870 uint32_t uShadow = 0;
5871 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5872 AssertRCReturn(rc, rc);
5873
5874 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5875 CPUMSetGuestCR0(pVCpu, uVal);
5876 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5877 }
5878
5879 HM_RESTORE_PREEMPT_IF_NEEDED();
5880 VMMRZCallRing3Enable(pVCpu);
5881 return VINF_SUCCESS;
5882}
5883
5884
5885/**
5886 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5887 *
5888 * @returns VBox status code.
5889 * @param pVCpu Pointer to the VMCPU.
5890 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5891 * out-of-sync. Make sure to update the required fields
5892 * before using them.
5893 *
5894 * @remarks No-long-jump zone!!!
5895 */
5896static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5897{
5898 NOREF(pMixedCtx);
5899
5900 int rc = VINF_SUCCESS;
5901 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5902 {
5903 uint32_t uVal = 0;
5904 uint32_t uShadow = 0;
5905 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5906 AssertRCReturn(rc, rc);
5907 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5908 AssertRCReturn(rc, rc);
5909
5910 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5911 CPUMSetGuestCR4(pVCpu, uVal);
5912 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5913 }
5914 return rc;
5915}
5916
5917
5918/**
5919 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5920 *
5921 * @returns VBox status code.
5922 * @param pVCpu Pointer to the VMCPU.
5923 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5924 * out-of-sync. Make sure to update the required fields
5925 * before using them.
5926 *
5927 * @remarks No-long-jump zone!!!
5928 */
5929static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5930{
5931 int rc = VINF_SUCCESS;
5932 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5933 {
5934 uint64_t u64Val = 0;
5935 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5936 AssertRCReturn(rc, rc);
5937
5938 pMixedCtx->rip = u64Val;
5939 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5940 }
5941 return rc;
5942}
5943
5944
5945/**
5946 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5947 *
5948 * @returns VBox status code.
5949 * @param pVCpu Pointer to the VMCPU.
5950 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5951 * out-of-sync. Make sure to update the required fields
5952 * before using them.
5953 *
5954 * @remarks No-long-jump zone!!!
5955 */
5956static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5957{
5958 int rc = VINF_SUCCESS;
5959 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5960 {
5961 uint64_t u64Val = 0;
5962 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5963 AssertRCReturn(rc, rc);
5964
5965 pMixedCtx->rsp = u64Val;
5966 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5967 }
5968 return rc;
5969}
5970
5971
5972/**
5973 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5974 *
5975 * @returns VBox status code.
5976 * @param pVCpu Pointer to the VMCPU.
5977 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5978 * out-of-sync. Make sure to update the required fields
5979 * before using them.
5980 *
5981 * @remarks No-long-jump zone!!!
5982 */
5983static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5984{
5985 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5986 {
5987 uint32_t uVal = 0;
5988 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5989 AssertRCReturn(rc, rc);
5990
5991 pMixedCtx->eflags.u32 = uVal;
5992 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5993 {
5994 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5995 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5996
5997 pMixedCtx->eflags.Bits.u1VM = 0;
5998 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5999 }
6000
6001 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6002 }
6003 return VINF_SUCCESS;
6004}
6005
6006
6007/**
6008 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6009 * guest-CPU context.
6010 */
6011DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6012{
6013 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6014 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6015 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6016 return rc;
6017}
6018
6019
6020/**
6021 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6022 * from the guest-state area in the VMCS.
6023 *
6024 * @param pVCpu Pointer to the VMCPU.
6025 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6026 * out-of-sync. Make sure to update the required fields
6027 * before using them.
6028 *
6029 * @remarks No-long-jump zone!!!
6030 */
6031static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6032{
6033 uint32_t uIntrState = 0;
6034 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6035 AssertRC(rc);
6036
6037 if (!uIntrState)
6038 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6039 else
6040 {
6041 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
6042 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6043 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6044 AssertRC(rc);
6045 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6046 AssertRC(rc);
6047
6048 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6049 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6050 }
6051}
6052
6053
6054/**
6055 * Saves the guest's activity state.
6056 *
6057 * @returns VBox status code.
6058 * @param pVCpu Pointer to the VMCPU.
6059 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6060 * out-of-sync. Make sure to update the required fields
6061 * before using them.
6062 *
6063 * @remarks No-long-jump zone!!!
6064 */
6065static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6066{
6067 NOREF(pMixedCtx);
6068 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6069 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6070 return VINF_SUCCESS;
6071}
6072
6073
6074/**
6075 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6076 * the current VMCS into the guest-CPU context.
6077 *
6078 * @returns VBox status code.
6079 * @param pVCpu Pointer to the VMCPU.
6080 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6081 * out-of-sync. Make sure to update the required fields
6082 * before using them.
6083 *
6084 * @remarks No-long-jump zone!!!
6085 */
6086static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6087{
6088 int rc = VINF_SUCCESS;
6089 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6090 {
6091 uint32_t u32Val = 0;
6092 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6093 pMixedCtx->SysEnter.cs = u32Val;
6094 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6095 }
6096
6097 uint64_t u64Val = 0;
6098 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6099 {
6100 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6101 pMixedCtx->SysEnter.eip = u64Val;
6102 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6103 }
6104 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6105 {
6106 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6107 pMixedCtx->SysEnter.esp = u64Val;
6108 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6109 }
6110 return rc;
6111}
6112
6113
6114/**
6115 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6116 * the CPU back into the guest-CPU context.
6117 *
6118 * @returns VBox status code.
6119 * @param pVCpu Pointer to the VMCPU.
6120 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6121 * out-of-sync. Make sure to update the required fields
6122 * before using them.
6123 *
6124 * @remarks No-long-jump zone!!!
6125 */
6126static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6127{
6128#if HC_ARCH_BITS == 64
6129 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6130 {
6131 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6132 VMMRZCallRing3Disable(pVCpu);
6133 HM_DISABLE_PREEMPT_IF_NEEDED();
6134
6135 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6136 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6137 {
6138 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6139 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6140 }
6141
6142 HM_RESTORE_PREEMPT_IF_NEEDED();
6143 VMMRZCallRing3Enable(pVCpu);
6144 }
6145 else
6146 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6147#else
6148 NOREF(pMixedCtx);
6149 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6150#endif
6151
6152 return VINF_SUCCESS;
6153}
6154
6155
6156/**
6157 * Saves the auto load/store'd guest MSRs from the current VMCS into
6158 * the guest-CPU context.
6159 *
6160 * @returns VBox status code.
6161 * @param pVCpu Pointer to the VMCPU.
6162 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6163 * out-of-sync. Make sure to update the required fields
6164 * before using them.
6165 *
6166 * @remarks No-long-jump zone!!!
6167 */
6168static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6169{
6170 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6171 return VINF_SUCCESS;
6172
6173 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6174 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6175 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6176 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6177 {
6178 switch (pMsr->u32Msr)
6179 {
6180 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6181 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6182 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6183 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6184 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6185#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6186 case MSR_K6_EFER:
6187 {
6188 if (HMVMX_IS_64BIT_HOST_MODE()) /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6189 break;
6190 }
6191#endif
6192 default:
6193 {
6194 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6195 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6196 }
6197 }
6198 }
6199
6200 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6201 return VINF_SUCCESS;
6202}
6203
6204
6205/**
6206 * Saves the guest control registers from the current VMCS into the guest-CPU
6207 * context.
6208 *
6209 * @returns VBox status code.
6210 * @param pVCpu Pointer to the VMCPU.
6211 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6212 * out-of-sync. Make sure to update the required fields
6213 * before using them.
6214 *
6215 * @remarks No-long-jump zone!!!
6216 */
6217static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6218{
6219 /* Guest CR0. Guest FPU. */
6220 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6221 AssertRCReturn(rc, rc);
6222
6223 /* Guest CR4. */
6224 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6225 AssertRCReturn(rc, rc);
6226
6227 /* Guest CR2 - updated always during the world-switch or in #PF. */
6228 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6229 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6230 {
6231 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6232 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6233
6234 PVM pVM = pVCpu->CTX_SUFF(pVM);
6235 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6236 || ( pVM->hm.s.fNestedPaging
6237 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6238 {
6239 uint64_t u64Val = 0;
6240 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6241 if (pMixedCtx->cr3 != u64Val)
6242 {
6243 CPUMSetGuestCR3(pVCpu, u64Val);
6244 if (VMMRZCallRing3IsEnabled(pVCpu))
6245 {
6246 PGMUpdateCR3(pVCpu, u64Val);
6247 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6248 }
6249 else
6250 {
6251 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6252 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6253 }
6254 }
6255
6256 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6257 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6258 {
6259 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6260 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6261 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6262 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6263
6264 if (VMMRZCallRing3IsEnabled(pVCpu))
6265 {
6266 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6267 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6268 }
6269 else
6270 {
6271 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6272 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6273 }
6274 }
6275 }
6276
6277 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6278 }
6279
6280 /*
6281 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6282 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6283 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6284 *
6285 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6286 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6287 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6288 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6289 *
6290 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6291 */
6292 if (VMMRZCallRing3IsEnabled(pVCpu))
6293 {
6294 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6295 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6296
6297 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6298 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6299
6300 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6301 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6302 }
6303
6304 return rc;
6305}
6306
6307
6308/**
6309 * Reads a guest segment register from the current VMCS into the guest-CPU
6310 * context.
6311 *
6312 * @returns VBox status code.
6313 * @param pVCpu Pointer to the VMCPU.
6314 * @param idxSel Index of the selector in the VMCS.
6315 * @param idxLimit Index of the segment limit in the VMCS.
6316 * @param idxBase Index of the segment base in the VMCS.
6317 * @param idxAccess Index of the access rights of the segment in the VMCS.
6318 * @param pSelReg Pointer to the segment selector.
6319 *
6320 * @remarks No-long-jump zone!!!
6321 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6322 * macro as that takes care of whether to read from the VMCS cache or
6323 * not.
6324 */
6325DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6326 PCPUMSELREG pSelReg)
6327{
6328 NOREF(pVCpu);
6329
6330 uint32_t u32Val = 0;
6331 int rc = VMXReadVmcs32(idxSel, &u32Val);
6332 AssertRCReturn(rc, rc);
6333 pSelReg->Sel = (uint16_t)u32Val;
6334 pSelReg->ValidSel = (uint16_t)u32Val;
6335 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6336
6337 rc = VMXReadVmcs32(idxLimit, &u32Val);
6338 AssertRCReturn(rc, rc);
6339 pSelReg->u32Limit = u32Val;
6340
6341 uint64_t u64Val = 0;
6342 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6343 AssertRCReturn(rc, rc);
6344 pSelReg->u64Base = u64Val;
6345
6346 rc = VMXReadVmcs32(idxAccess, &u32Val);
6347 AssertRCReturn(rc, rc);
6348 pSelReg->Attr.u = u32Val;
6349
6350 /*
6351 * If VT-x marks the segment as unusable, most other bits remain undefined:
6352 * - For CS the L, D and G bits have meaning.
6353 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6354 * - For the remaining data segments no bits are defined.
6355 *
6356 * The present bit and the unusable bit has been observed to be set at the
6357 * same time (the selector was supposed to invalid as we started executing
6358 * a V8086 interrupt in ring-0).
6359 *
6360 * What should be important for the rest of the VBox code, is that the P bit is
6361 * cleared. Some of the other VBox code recognizes the unusable bit, but
6362 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6363 * safe side here, we'll strip off P and other bits we don't care about. If
6364 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6365 *
6366 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6367 */
6368 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6369 {
6370 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6371
6372 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6373 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6374 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6375
6376 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6377#ifdef DEBUG_bird
6378 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6379 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6380 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6381#endif
6382 }
6383 return VINF_SUCCESS;
6384}
6385
6386
6387#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6388# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6389 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6390 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6391#else
6392# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6393 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6394 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6395#endif
6396
6397
6398/**
6399 * Saves the guest segment registers from the current VMCS into the guest-CPU
6400 * context.
6401 *
6402 * @returns VBox status code.
6403 * @param pVCpu Pointer to the VMCPU.
6404 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6405 * out-of-sync. Make sure to update the required fields
6406 * before using them.
6407 *
6408 * @remarks No-long-jump zone!!!
6409 */
6410static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6411{
6412 /* Guest segment registers. */
6413 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6414 {
6415 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6416 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6417 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6418 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6419 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6420 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6421 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6422
6423 /* Restore segment attributes for real-on-v86 mode hack. */
6424 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6425 {
6426 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6427 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6428 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6429 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6430 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6431 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6432 }
6433 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6434 }
6435
6436 return VINF_SUCCESS;
6437}
6438
6439
6440/**
6441 * Saves the guest descriptor table registers and task register from the current
6442 * VMCS into the guest-CPU context.
6443 *
6444 * @returns VBox status code.
6445 * @param pVCpu Pointer to the VMCPU.
6446 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6447 * out-of-sync. Make sure to update the required fields
6448 * before using them.
6449 *
6450 * @remarks No-long-jump zone!!!
6451 */
6452static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6453{
6454 int rc = VINF_SUCCESS;
6455
6456 /* Guest LDTR. */
6457 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6458 {
6459 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6460 AssertRCReturn(rc, rc);
6461 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6462 }
6463
6464 /* Guest GDTR. */
6465 uint64_t u64Val = 0;
6466 uint32_t u32Val = 0;
6467 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6468 {
6469 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6470 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6471 pMixedCtx->gdtr.pGdt = u64Val;
6472 pMixedCtx->gdtr.cbGdt = u32Val;
6473 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6474 }
6475
6476 /* Guest IDTR. */
6477 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6478 {
6479 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6480 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6481 pMixedCtx->idtr.pIdt = u64Val;
6482 pMixedCtx->idtr.cbIdt = u32Val;
6483 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6484 }
6485
6486 /* Guest TR. */
6487 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6488 {
6489 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6490 AssertRCReturn(rc, rc);
6491
6492 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6493 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6494 {
6495 rc = VMXLOCAL_READ_SEG(TR, tr);
6496 AssertRCReturn(rc, rc);
6497 }
6498 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6499 }
6500 return rc;
6501}
6502
6503#undef VMXLOCAL_READ_SEG
6504
6505
6506/**
6507 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6508 * context.
6509 *
6510 * @returns VBox status code.
6511 * @param pVCpu Pointer to the VMCPU.
6512 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6513 * out-of-sync. Make sure to update the required fields
6514 * before using them.
6515 *
6516 * @remarks No-long-jump zone!!!
6517 */
6518static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6519{
6520 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6521 {
6522 if (!pVCpu->hm.s.fUsingHyperDR7)
6523 {
6524 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6525 uint32_t u32Val;
6526 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6527 pMixedCtx->dr[7] = u32Val;
6528 }
6529
6530 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6531 }
6532 return VINF_SUCCESS;
6533}
6534
6535
6536/**
6537 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6538 *
6539 * @returns VBox status code.
6540 * @param pVCpu Pointer to the VMCPU.
6541 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6542 * out-of-sync. Make sure to update the required fields
6543 * before using them.
6544 *
6545 * @remarks No-long-jump zone!!!
6546 */
6547static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6548{
6549 NOREF(pMixedCtx);
6550
6551 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6552 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6553 return VINF_SUCCESS;
6554}
6555
6556
6557/**
6558 * Saves the entire guest state from the currently active VMCS into the
6559 * guest-CPU context. This essentially VMREADs all guest-data.
6560 *
6561 * @returns VBox status code.
6562 * @param pVCpu Pointer to the VMCPU.
6563 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6564 * out-of-sync. Make sure to update the required fields
6565 * before using them.
6566 */
6567static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6568{
6569 Assert(pVCpu);
6570 Assert(pMixedCtx);
6571
6572 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6573 return VINF_SUCCESS;
6574
6575 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6576 again on the ring-3 callback path, there is no real need to. */
6577 if (VMMRZCallRing3IsEnabled(pVCpu))
6578 VMMR0LogFlushDisable(pVCpu);
6579 else
6580 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6581 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6582
6583 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6584 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6585
6586 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6587 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6588
6589 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6590 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6591
6592 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6593 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6594
6595 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6596 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6597
6598 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6599 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6600
6601 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6602 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6603
6604 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6605 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6606
6607 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6608 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6609
6610 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6611 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6612
6613 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6614 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6615
6616 if (VMMRZCallRing3IsEnabled(pVCpu))
6617 VMMR0LogFlushEnable(pVCpu);
6618
6619 return rc;
6620}
6621
6622
6623/**
6624 * Check per-VM and per-VCPU force flag actions that require us to go back to
6625 * ring-3 for one reason or another.
6626 *
6627 * @returns VBox status code (information status code included).
6628 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6629 * ring-3.
6630 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6631 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6632 * interrupts)
6633 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6634 * all EMTs to be in ring-3.
6635 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6636 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6637 * to the EM loop.
6638 *
6639 * @param pVM Pointer to the VM.
6640 * @param pVCpu Pointer to the VMCPU.
6641 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6642 * out-of-sync. Make sure to update the required fields
6643 * before using them.
6644 */
6645static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6646{
6647 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6648
6649 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6650 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6651 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6652 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6653 {
6654 /* We need the control registers now, make sure the guest-CPU context is updated. */
6655 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6656 AssertRCReturn(rc3, rc3);
6657
6658 /* Pending HM CR3 sync. */
6659 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6660 {
6661 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6662 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6663 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6664 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6665 }
6666
6667 /* Pending HM PAE PDPEs. */
6668 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6669 {
6670 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6671 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6672 }
6673
6674 /* Pending PGM C3 sync. */
6675 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6676 {
6677 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6678 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6679 if (rc2 != VINF_SUCCESS)
6680 {
6681 AssertRC(rc2);
6682 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6683 return rc2;
6684 }
6685 }
6686
6687 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6688 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6689 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6690 {
6691 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6692 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6693 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6694 return rc2;
6695 }
6696
6697 /* Pending VM request packets, such as hardware interrupts. */
6698 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6699 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6700 {
6701 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6702 return VINF_EM_PENDING_REQUEST;
6703 }
6704
6705 /* Pending PGM pool flushes. */
6706 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6707 {
6708 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6709 return VINF_PGM_POOL_FLUSH_PENDING;
6710 }
6711
6712 /* Pending DMA requests. */
6713 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6714 {
6715 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6716 return VINF_EM_RAW_TO_R3;
6717 }
6718 }
6719
6720 return VINF_SUCCESS;
6721}
6722
6723
6724/**
6725 * Converts any TRPM trap into a pending HM event. This is typically used when
6726 * entering from ring-3 (not longjmp returns).
6727 *
6728 * @param pVCpu Pointer to the VMCPU.
6729 */
6730static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6731{
6732 Assert(TRPMHasTrap(pVCpu));
6733 Assert(!pVCpu->hm.s.Event.fPending);
6734
6735 uint8_t uVector;
6736 TRPMEVENT enmTrpmEvent;
6737 RTGCUINT uErrCode;
6738 RTGCUINTPTR GCPtrFaultAddress;
6739 uint8_t cbInstr;
6740
6741 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6742 AssertRC(rc);
6743
6744 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6745 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6746 if (enmTrpmEvent == TRPM_TRAP)
6747 {
6748 switch (uVector)
6749 {
6750 case X86_XCPT_BP:
6751 case X86_XCPT_OF:
6752 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6753 break;
6754
6755 case X86_XCPT_PF:
6756 case X86_XCPT_DF:
6757 case X86_XCPT_TS:
6758 case X86_XCPT_NP:
6759 case X86_XCPT_SS:
6760 case X86_XCPT_GP:
6761 case X86_XCPT_AC:
6762 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6763 /* no break! */
6764 default:
6765 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6766 break;
6767 }
6768 }
6769 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6770 {
6771 if (uVector == X86_XCPT_NMI)
6772 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6773 else
6774 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6775 }
6776 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6777 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6778 else
6779 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6780
6781 rc = TRPMResetTrap(pVCpu);
6782 AssertRC(rc);
6783 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6784 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6785
6786 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6787 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6788}
6789
6790
6791/**
6792 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6793 * VT-x to execute any instruction.
6794 *
6795 * @param pvCpu Pointer to the VMCPU.
6796 */
6797static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6798{
6799 Assert(pVCpu->hm.s.Event.fPending);
6800
6801 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6802 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6803 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6804 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6805
6806 /* If a trap was already pending, we did something wrong! */
6807 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6808
6809 TRPMEVENT enmTrapType;
6810 switch (uVectorType)
6811 {
6812 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6813 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6814 enmTrapType = TRPM_HARDWARE_INT;
6815 break;
6816
6817 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6818 enmTrapType = TRPM_SOFTWARE_INT;
6819 break;
6820
6821 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6822 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6823 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6824 enmTrapType = TRPM_TRAP;
6825 break;
6826
6827 default:
6828 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6829 enmTrapType = TRPM_32BIT_HACK;
6830 break;
6831 }
6832
6833 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6834
6835 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6836 AssertRC(rc);
6837
6838 if (fErrorCodeValid)
6839 TRPMSetErrorCode(pVCpu, uErrorCode);
6840
6841 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6842 && uVector == X86_XCPT_PF)
6843 {
6844 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6845 }
6846 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6847 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6848 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6849 {
6850 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6851 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6852 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6853 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6854 }
6855 pVCpu->hm.s.Event.fPending = false;
6856}
6857
6858
6859/**
6860 * Does the necessary state syncing before returning to ring-3 for any reason
6861 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6862 *
6863 * @returns VBox status code.
6864 * @param pVM Pointer to the VM.
6865 * @param pVCpu Pointer to the VMCPU.
6866 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6867 * be out-of-sync. Make sure to update the required
6868 * fields before using them.
6869 * @param fSaveGuestState Whether to save the guest state or not.
6870 *
6871 * @remarks No-long-jmp zone!!!
6872 */
6873static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6874{
6875 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6876 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6877
6878 RTCPUID idCpu = RTMpCpuId();
6879 Log4Func(("HostCpuId=%u\n", idCpu));
6880
6881 /*
6882 * !!! IMPORTANT !!!
6883 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6884 */
6885
6886 /* Save the guest state if necessary. */
6887 if ( fSaveGuestState
6888 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6889 {
6890 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6891 AssertRCReturn(rc, rc);
6892 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6893 }
6894
6895 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6896 if (CPUMIsGuestFPUStateActive(pVCpu))
6897 {
6898 /* We shouldn't reload CR0 without saving it first. */
6899 if (!fSaveGuestState)
6900 {
6901 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6902 AssertRCReturn(rc, rc);
6903 }
6904 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6905 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6906 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6907 }
6908
6909 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6910#ifdef VBOX_STRICT
6911 if (CPUMIsHyperDebugStateActive(pVCpu))
6912 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6913#endif
6914 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6915 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6916 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6917 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6918
6919#if HC_ARCH_BITS == 64
6920 /* Restore host-state bits that VT-x only restores partially. */
6921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6923 {
6924 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6925 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6926 }
6927 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6928#endif
6929
6930#if HC_ARCH_BITS == 64
6931 /* Restore the host MSRs as we're leaving VT-x context. */
6932 if ( pVM->hm.s.fAllow64BitGuests
6933 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6934 {
6935 /* We shouldn't reload the guest MSRs without saving it first. */
6936 if (!fSaveGuestState)
6937 {
6938 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6939 AssertRCReturn(rc, rc);
6940 }
6941 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6942 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6943 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6944 }
6945#endif
6946
6947 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6948 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6949
6950 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6951 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6952 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6953 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6954 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6955 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6956 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6957 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6958
6959 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6960
6961 /** @todo This partially defeats the purpose of having preemption hooks.
6962 * The problem is, deregistering the hooks should be moved to a place that
6963 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6964 * context.
6965 */
6966 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6967 {
6968 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6969 AssertRCReturn(rc, rc);
6970
6971 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6972 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6973 }
6974 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6975 NOREF(idCpu);
6976
6977 return VINF_SUCCESS;
6978}
6979
6980
6981/**
6982 * Leaves the VT-x session.
6983 *
6984 * @returns VBox status code.
6985 * @param pVM Pointer to the VM.
6986 * @param pVCpu Pointer to the VMCPU.
6987 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6988 * out-of-sync. Make sure to update the required fields
6989 * before using them.
6990 *
6991 * @remarks No-long-jmp zone!!!
6992 */
6993DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6994{
6995 HM_DISABLE_PREEMPT_IF_NEEDED();
6996 HMVMX_ASSERT_CPU_SAFE();
6997 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6998 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6999
7000 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7001 and done this from the VMXR0ThreadCtxCallback(). */
7002 if (!pVCpu->hm.s.fLeaveDone)
7003 {
7004 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7005 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7006 pVCpu->hm.s.fLeaveDone = true;
7007 }
7008
7009 /*
7010 * !!! IMPORTANT !!!
7011 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7012 */
7013
7014 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7015 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7016 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7017 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7018 VMMR0ThreadCtxHooksDeregister(pVCpu);
7019
7020 /* Leave HM context. This takes care of local init (term). */
7021 int rc = HMR0LeaveCpu(pVCpu);
7022
7023 HM_RESTORE_PREEMPT_IF_NEEDED();
7024
7025 return rc;
7026}
7027
7028
7029/**
7030 * Does the necessary state syncing before doing a longjmp to ring-3.
7031 *
7032 * @returns VBox status code.
7033 * @param pVM Pointer to the VM.
7034 * @param pVCpu Pointer to the VMCPU.
7035 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7036 * out-of-sync. Make sure to update the required fields
7037 * before using them.
7038 *
7039 * @remarks No-long-jmp zone!!!
7040 */
7041DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7042{
7043 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7044}
7045
7046
7047/**
7048 * Take necessary actions before going back to ring-3.
7049 *
7050 * An action requires us to go back to ring-3. This function does the necessary
7051 * steps before we can safely return to ring-3. This is not the same as longjmps
7052 * to ring-3, this is voluntary and prepares the guest so it may continue
7053 * executing outside HM (recompiler/IEM).
7054 *
7055 * @returns VBox status code.
7056 * @param pVM Pointer to the VM.
7057 * @param pVCpu Pointer to the VMCPU.
7058 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7059 * out-of-sync. Make sure to update the required fields
7060 * before using them.
7061 * @param rcExit The reason for exiting to ring-3. Can be
7062 * VINF_VMM_UNKNOWN_RING3_CALL.
7063 */
7064static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7065{
7066 Assert(pVM);
7067 Assert(pVCpu);
7068 Assert(pMixedCtx);
7069 HMVMX_ASSERT_PREEMPT_SAFE();
7070
7071 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7072 {
7073 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7074 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7075 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7076 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7077 }
7078
7079 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7080 VMMRZCallRing3Disable(pVCpu);
7081 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7082
7083 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7084 if (pVCpu->hm.s.Event.fPending)
7085 {
7086 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7087 Assert(!pVCpu->hm.s.Event.fPending);
7088 }
7089
7090 /* Save guest state and restore host state bits. */
7091 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7092 AssertRCReturn(rc, rc);
7093 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7094
7095 /* Sync recompiler state. */
7096 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7097 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7098 | CPUM_CHANGED_LDTR
7099 | CPUM_CHANGED_GDTR
7100 | CPUM_CHANGED_IDTR
7101 | CPUM_CHANGED_TR
7102 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7103 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7104 if ( pVM->hm.s.fNestedPaging
7105 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7106 {
7107 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7108 }
7109
7110 Assert(!pVCpu->hm.s.fClearTrapFlag);
7111
7112 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7113 if (rcExit != VINF_EM_RAW_INTERRUPT)
7114 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7115
7116 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7117
7118 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7119 VMMRZCallRing3RemoveNotification(pVCpu);
7120 VMMRZCallRing3Enable(pVCpu);
7121
7122 return rc;
7123}
7124
7125
7126/**
7127 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7128 * longjump to ring-3 and possibly get preempted.
7129 *
7130 * @returns VBox status code.
7131 * @param pVCpu Pointer to the VMCPU.
7132 * @param enmOperation The operation causing the ring-3 longjump.
7133 * @param pvUser Opaque pointer to the guest-CPU context. The data
7134 * may be out-of-sync. Make sure to update the required
7135 * fields before using them.
7136 */
7137DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7138{
7139 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7140 {
7141 /*
7142 * !!! IMPORTANT !!!
7143 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7144 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7145 */
7146 VMMRZCallRing3RemoveNotification(pVCpu);
7147 VMMRZCallRing3Disable(pVCpu);
7148 HM_DISABLE_PREEMPT_IF_NEEDED();
7149
7150 PVM pVM = pVCpu->CTX_SUFF(pVM);
7151 if (CPUMIsGuestFPUStateActive(pVCpu))
7152 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7153
7154 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7155
7156#if HC_ARCH_BITS == 64
7157 /* Restore host-state bits that VT-x only restores partially. */
7158 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7159 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7160 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7161 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7162
7163 /* Restore the host MSRs as we're leaving VT-x context. */
7164 if ( pVM->hm.s.fAllow64BitGuests
7165 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7166 {
7167 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7168 }
7169#endif
7170 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7171 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7172 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7173 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7174 {
7175 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7176 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7177 }
7178
7179 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7180 VMMR0ThreadCtxHooksDeregister(pVCpu);
7181
7182 HMR0LeaveCpu(pVCpu);
7183 HM_RESTORE_PREEMPT_IF_NEEDED();
7184 return VINF_SUCCESS;
7185 }
7186
7187 Assert(pVCpu);
7188 Assert(pvUser);
7189 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7190 HMVMX_ASSERT_PREEMPT_SAFE();
7191
7192 VMMRZCallRing3Disable(pVCpu);
7193 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7194
7195 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7196 enmOperation));
7197
7198 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7199 AssertRCReturn(rc, rc);
7200
7201 VMMRZCallRing3Enable(pVCpu);
7202 return VINF_SUCCESS;
7203}
7204
7205
7206/**
7207 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7208 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7209 *
7210 * @param pVCpu Pointer to the VMCPU.
7211 */
7212DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7213{
7214 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7215 {
7216 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7217 {
7218 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7219 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7220 AssertRC(rc);
7221 Log4(("Setup interrupt-window exiting\n"));
7222 }
7223 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7224}
7225
7226
7227/**
7228 * Clears the interrupt-window exiting control in the VMCS.
7229 *
7230 * @param pVCpu Pointer to the VMCPU.
7231 */
7232DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7233{
7234 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7235 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7236 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7237 AssertRC(rc);
7238 Log4(("Cleared interrupt-window exiting\n"));
7239}
7240
7241
7242/**
7243 * Evaluates the event to be delivered to the guest and sets it as the pending
7244 * event.
7245 *
7246 * @param pVCpu Pointer to the VMCPU.
7247 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7248 * out-of-sync. Make sure to update the required fields
7249 * before using them.
7250 */
7251static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7252{
7253 Assert(!pVCpu->hm.s.Event.fPending);
7254
7255 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7256 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7257 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7258 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7259
7260 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7261 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7262 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7263 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7264 Assert(!TRPMHasTrap(pVCpu));
7265
7266 /** @todo SMI. SMIs take priority over NMIs. */
7267 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7268 {
7269 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7270 if ( !fBlockMovSS
7271 && !fBlockSti)
7272 {
7273 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7274 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7275 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7276 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7277
7278 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7279 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7280 }
7281 else
7282 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7283 }
7284 /*
7285 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7286 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7287 */
7288 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7289 && !pVCpu->hm.s.fSingleInstruction)
7290 {
7291 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7292 AssertRC(rc);
7293 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7294 if ( !fBlockInt
7295 && !fBlockSti
7296 && !fBlockMovSS)
7297 {
7298 uint8_t u8Interrupt;
7299 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7300 if (RT_SUCCESS(rc))
7301 {
7302 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7303 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7304 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7305
7306 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7307 }
7308 else
7309 {
7310 /** @todo Does this actually happen? If not turn it into an assertion. */
7311 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7312 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7313 }
7314 }
7315 else
7316 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7317 }
7318}
7319
7320
7321/**
7322 * Sets a pending-debug exception to be delivered to the guest if the guest is
7323 * single-stepping.
7324 *
7325 * @param pVCpu Pointer to the VMCPU.
7326 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7327 * out-of-sync. Make sure to update the required fields
7328 * before using them.
7329 */
7330DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7331{
7332 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7333 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7334 {
7335 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7336 AssertRC(rc);
7337 }
7338}
7339
7340
7341/**
7342 * Injects any pending events into the guest if the guest is in a state to
7343 * receive them.
7344 *
7345 * @returns VBox status code (informational status codes included).
7346 * @param pVCpu Pointer to the VMCPU.
7347 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7348 * out-of-sync. Make sure to update the required fields
7349 * before using them.
7350 */
7351static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7352{
7353 HMVMX_ASSERT_PREEMPT_SAFE();
7354 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7355
7356 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7357 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7358 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7359 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7360
7361 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7362 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7363 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7364 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7365 Assert(!TRPMHasTrap(pVCpu));
7366
7367 int rc = VINF_SUCCESS;
7368 if (pVCpu->hm.s.Event.fPending)
7369 {
7370 /*
7371 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7372 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7373 * ended up enabling interrupts outside VT-x.
7374 */
7375 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7376 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7377 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7378 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7379 {
7380 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7381 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7382 }
7383#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
7384 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7385 {
7386 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7387 AssertRCReturn(rc, rc);
7388 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7389 Assert(!fBlockInt);
7390 Assert(!fBlockSti);
7391 Assert(!fBlockMovSS);
7392 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7393 }
7394 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7395 {
7396 Assert(!fBlockSti);
7397 Assert(!fBlockMovSS);
7398 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7399 }
7400#endif
7401 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7402 (uint8_t)uIntType));
7403 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7404 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7405 AssertRCReturn(rc, rc);
7406
7407 /* Update the interruptibility-state as it could have been changed by
7408 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7409 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7410 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7411
7412#ifdef VBOX_WITH_STATISTICS
7413 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7414 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7415 else
7416 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7417#endif
7418 }
7419
7420 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7421 if ( fBlockSti
7422 || fBlockMovSS)
7423 {
7424 if ( !pVCpu->hm.s.fSingleInstruction
7425 && !DBGFIsStepping(pVCpu))
7426 {
7427 /*
7428 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7429 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7430 * See Intel spec. 27.3.4 "Saving Non-Register State".
7431 */
7432 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7433 AssertRCReturn(rc2, rc2);
7434 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7435 }
7436 else if (pMixedCtx->eflags.Bits.u1TF)
7437 {
7438 /*
7439 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7440 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7441 */
7442 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7443 uIntrState = 0;
7444 }
7445 }
7446
7447 /*
7448 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7449 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7450 */
7451 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7452 AssertRC(rc2);
7453
7454 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7455 NOREF(fBlockMovSS); NOREF(fBlockSti);
7456 return rc;
7457}
7458
7459
7460/**
7461 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7462 *
7463 * @param pVCpu Pointer to the VMCPU.
7464 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7465 * out-of-sync. Make sure to update the required fields
7466 * before using them.
7467 */
7468DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7469{
7470 NOREF(pMixedCtx);
7471 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7472 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7473}
7474
7475
7476/**
7477 * Injects a double-fault (#DF) exception into the VM.
7478 *
7479 * @returns VBox status code (informational status code included).
7480 * @param pVCpu Pointer to the VMCPU.
7481 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7482 * out-of-sync. Make sure to update the required fields
7483 * before using them.
7484 */
7485DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7486{
7487 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7488 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7489 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7490 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7491 puIntrState);
7492}
7493
7494
7495/**
7496 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7497 *
7498 * @param pVCpu Pointer to the VMCPU.
7499 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7500 * out-of-sync. Make sure to update the required fields
7501 * before using them.
7502 */
7503DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7504{
7505 NOREF(pMixedCtx);
7506 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7507 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7508 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7509}
7510
7511
7512/**
7513 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7514 *
7515 * @param pVCpu Pointer to the VMCPU.
7516 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7517 * out-of-sync. Make sure to update the required fields
7518 * before using them.
7519 * @param cbInstr The value of RIP that is to be pushed on the guest
7520 * stack.
7521 */
7522DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7523{
7524 NOREF(pMixedCtx);
7525 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7526 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7527 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7528}
7529
7530
7531/**
7532 * Injects a general-protection (#GP) fault into the VM.
7533 *
7534 * @returns VBox status code (informational status code included).
7535 * @param pVCpu Pointer to the VMCPU.
7536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7537 * out-of-sync. Make sure to update the required fields
7538 * before using them.
7539 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7540 * mode, i.e. in real-mode it's not valid).
7541 * @param u32ErrorCode The error code associated with the #GP.
7542 */
7543DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7544 uint32_t *puIntrState)
7545{
7546 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7547 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7548 if (fErrorCodeValid)
7549 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7550 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7551 puIntrState);
7552}
7553
7554
7555/**
7556 * Sets a general-protection (#GP) exception as pending-for-injection into the
7557 * VM.
7558 *
7559 * @param pVCpu Pointer to the VMCPU.
7560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7561 * out-of-sync. Make sure to update the required fields
7562 * before using them.
7563 * @param u32ErrorCode The error code associated with the #GP.
7564 */
7565DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7566{
7567 NOREF(pMixedCtx);
7568 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7569 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7570 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7571 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7572}
7573
7574
7575/**
7576 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7577 *
7578 * @param pVCpu Pointer to the VMCPU.
7579 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7580 * out-of-sync. Make sure to update the required fields
7581 * before using them.
7582 * @param uVector The software interrupt vector number.
7583 * @param cbInstr The value of RIP that is to be pushed on the guest
7584 * stack.
7585 */
7586DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7587{
7588 NOREF(pMixedCtx);
7589 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7590 if ( uVector == X86_XCPT_BP
7591 || uVector == X86_XCPT_OF)
7592 {
7593 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7594 }
7595 else
7596 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7597 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7598}
7599
7600
7601/**
7602 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7603 * stack.
7604 *
7605 * @returns VBox status code (information status code included).
7606 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7607 * @param pVM Pointer to the VM.
7608 * @param pMixedCtx Pointer to the guest-CPU context.
7609 * @param uValue The value to push to the guest stack.
7610 */
7611DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7612{
7613 /*
7614 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7615 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7616 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7617 */
7618 if (pMixedCtx->sp == 1)
7619 return VINF_EM_RESET;
7620 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7621 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7622 AssertRCReturn(rc, rc);
7623 return rc;
7624}
7625
7626
7627/**
7628 * Injects an event into the guest upon VM-entry by updating the relevant fields
7629 * in the VM-entry area in the VMCS.
7630 *
7631 * @returns VBox status code (informational error codes included).
7632 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7633 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7634 *
7635 * @param pVCpu Pointer to the VMCPU.
7636 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7637 * be out-of-sync. Make sure to update the required
7638 * fields before using them.
7639 * @param u64IntInfo The VM-entry interruption-information field.
7640 * @param cbInstr The VM-entry instruction length in bytes (for
7641 * software interrupts, exceptions and privileged
7642 * software exceptions).
7643 * @param u32ErrCode The VM-entry exception error code.
7644 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7645 * @param puIntrState Pointer to the current guest interruptibility-state.
7646 * This interruptibility-state will be updated if
7647 * necessary. This cannot not be NULL.
7648 *
7649 * @remarks Requires CR0!
7650 * @remarks No-long-jump zone!!!
7651 */
7652static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7653 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7654{
7655 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7656 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7657 Assert(puIntrState);
7658 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7659
7660 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7661 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7662
7663#ifdef VBOX_STRICT
7664 /* Validate the error-code-valid bit for hardware exceptions. */
7665 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7666 {
7667 switch (uVector)
7668 {
7669 case X86_XCPT_PF:
7670 case X86_XCPT_DF:
7671 case X86_XCPT_TS:
7672 case X86_XCPT_NP:
7673 case X86_XCPT_SS:
7674 case X86_XCPT_GP:
7675 case X86_XCPT_AC:
7676 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7677 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7678 /* fallthru */
7679 default:
7680 break;
7681 }
7682 }
7683#endif
7684
7685 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7686 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7687 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7688
7689 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7690
7691 /* We require CR0 to check if the guest is in real-mode. */
7692 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7693 AssertRCReturn(rc, rc);
7694
7695 /*
7696 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7697 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7698 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7699 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7700 */
7701 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7702 {
7703 PVM pVM = pVCpu->CTX_SUFF(pVM);
7704 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7705 {
7706 Assert(PDMVmmDevHeapIsEnabled(pVM));
7707 Assert(pVM->hm.s.vmx.pRealModeTSS);
7708
7709 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7710 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7711 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7712 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7713 AssertRCReturn(rc, rc);
7714 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7715
7716 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7717 const size_t cbIdtEntry = sizeof(X86IDTR16);
7718 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7719 {
7720 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7721 if (uVector == X86_XCPT_DF)
7722 return VINF_EM_RESET;
7723 else if (uVector == X86_XCPT_GP)
7724 {
7725 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7726 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7727 }
7728
7729 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7730 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7731 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7732 }
7733
7734 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7735 uint16_t uGuestIp = pMixedCtx->ip;
7736 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7737 {
7738 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7739 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7740 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7741 }
7742 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7743 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7744
7745 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7746 X86IDTR16 IdtEntry;
7747 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7748 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7749 AssertRCReturn(rc, rc);
7750
7751 /* Construct the stack frame for the interrupt/exception handler. */
7752 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7753 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7754 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7755 AssertRCReturn(rc, rc);
7756
7757 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7758 if (rc == VINF_SUCCESS)
7759 {
7760 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7761 pMixedCtx->rip = IdtEntry.offSel;
7762 pMixedCtx->cs.Sel = IdtEntry.uSel;
7763 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7764 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7765 && uVector == X86_XCPT_PF)
7766 {
7767 pMixedCtx->cr2 = GCPtrFaultAddress;
7768 }
7769
7770 /* If any other guest-state bits are changed here, make sure to update
7771 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7772 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7773 | HM_CHANGED_GUEST_RIP
7774 | HM_CHANGED_GUEST_RFLAGS
7775 | HM_CHANGED_GUEST_RSP);
7776
7777 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7778 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7779 {
7780 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7781 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7782 Log4(("Clearing inhibition due to STI.\n"));
7783 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7784 }
7785 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7786
7787 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7788 it, if we are returning to ring-3 before executing guest code. */
7789 pVCpu->hm.s.Event.fPending = false;
7790 }
7791 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7792 return rc;
7793 }
7794 else
7795 {
7796 /*
7797 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7798 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7799 */
7800 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7801 }
7802 }
7803
7804 /* Validate. */
7805 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7806 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7807 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7808
7809 /* Inject. */
7810 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7811 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7812 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7813 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7814
7815 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7816 && uVector == X86_XCPT_PF)
7817 {
7818 pMixedCtx->cr2 = GCPtrFaultAddress;
7819 }
7820
7821 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7822 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7823
7824 AssertRCReturn(rc, rc);
7825 return rc;
7826}
7827
7828
7829/**
7830 * Clears the interrupt-window exiting control in the VMCS and if necessary
7831 * clears the current event in the VMCS as well.
7832 *
7833 * @returns VBox status code.
7834 * @param pVCpu Pointer to the VMCPU.
7835 *
7836 * @remarks Use this function only to clear events that have not yet been
7837 * delivered to the guest but are injected in the VMCS!
7838 * @remarks No-long-jump zone!!!
7839 */
7840static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7841{
7842 int rc;
7843 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7844
7845 /* Clear interrupt-window exiting control. */
7846 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7847 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7848
7849 if (!pVCpu->hm.s.Event.fPending)
7850 return;
7851
7852#ifdef VBOX_STRICT
7853 uint32_t u32EntryInfo;
7854 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7855 AssertRC(rc);
7856 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7857#endif
7858
7859 /* Clear the entry-interruption field (including the valid bit). */
7860 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7861 AssertRC(rc);
7862
7863 /* Clear the pending debug exception field. */
7864 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7865 AssertRC(rc);
7866
7867 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
7868 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
7869}
7870
7871
7872/**
7873 * Enters the VT-x session.
7874 *
7875 * @returns VBox status code.
7876 * @param pVM Pointer to the VM.
7877 * @param pVCpu Pointer to the VMCPU.
7878 * @param pCpu Pointer to the CPU info struct.
7879 */
7880VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7881{
7882 AssertPtr(pVM);
7883 AssertPtr(pVCpu);
7884 Assert(pVM->hm.s.vmx.fSupported);
7885 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7886 NOREF(pCpu); NOREF(pVM);
7887
7888 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7889 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7890
7891#ifdef VBOX_STRICT
7892 /* Make sure we're in VMX root mode. */
7893 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7894 if (!(u32HostCR4 & X86_CR4_VMXE))
7895 {
7896 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7897 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7898 }
7899#endif
7900
7901 /*
7902 * Load the VCPU's VMCS as the current (and active) one.
7903 */
7904 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7905 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7906 if (RT_FAILURE(rc))
7907 return rc;
7908
7909 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7910 pVCpu->hm.s.fLeaveDone = false;
7911 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7912
7913 return VINF_SUCCESS;
7914}
7915
7916
7917/**
7918 * The thread-context callback (only on platforms which support it).
7919 *
7920 * @param enmEvent The thread-context event.
7921 * @param pVCpu Pointer to the VMCPU.
7922 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7923 * @thread EMT(pVCpu)
7924 */
7925VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7926{
7927 NOREF(fGlobalInit);
7928
7929 switch (enmEvent)
7930 {
7931 case RTTHREADCTXEVENT_PREEMPTING:
7932 {
7933 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7934 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7935 VMCPU_ASSERT_EMT(pVCpu);
7936
7937 PVM pVM = pVCpu->CTX_SUFF(pVM);
7938 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7939
7940 /* No longjmps (logger flushes, locks) in this fragile context. */
7941 VMMRZCallRing3Disable(pVCpu);
7942 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7943
7944 /*
7945 * Restore host-state (FPU, debug etc.)
7946 */
7947 if (!pVCpu->hm.s.fLeaveDone)
7948 {
7949 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7950 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7951 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7952 pVCpu->hm.s.fLeaveDone = true;
7953 }
7954
7955 /* Leave HM context, takes care of local init (term). */
7956 int rc = HMR0LeaveCpu(pVCpu);
7957 AssertRC(rc); NOREF(rc);
7958
7959 /* Restore longjmp state. */
7960 VMMRZCallRing3Enable(pVCpu);
7961 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7962 break;
7963 }
7964
7965 case RTTHREADCTXEVENT_RESUMED:
7966 {
7967 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7968 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7969 VMCPU_ASSERT_EMT(pVCpu);
7970
7971 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7972 VMMRZCallRing3Disable(pVCpu);
7973 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7974
7975 /* Initialize the bare minimum state required for HM. This takes care of
7976 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7977 int rc = HMR0EnterCpu(pVCpu);
7978 AssertRC(rc);
7979 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7980
7981 /* Load the active VMCS as the current one. */
7982 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7983 {
7984 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7985 AssertRC(rc); NOREF(rc);
7986 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7987 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7988 }
7989 pVCpu->hm.s.fLeaveDone = false;
7990
7991 /* Restore longjmp state. */
7992 VMMRZCallRing3Enable(pVCpu);
7993 break;
7994 }
7995
7996 default:
7997 break;
7998 }
7999}
8000
8001
8002/**
8003 * Saves the host state in the VMCS host-state.
8004 * Sets up the VM-exit MSR-load area.
8005 *
8006 * The CPU state will be loaded from these fields on every successful VM-exit.
8007 *
8008 * @returns VBox status code.
8009 * @param pVM Pointer to the VM.
8010 * @param pVCpu Pointer to the VMCPU.
8011 *
8012 * @remarks No-long-jump zone!!!
8013 */
8014static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8015{
8016 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8017
8018 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8019 return VINF_SUCCESS;
8020
8021 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8022 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8023
8024 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8025 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8026
8027 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8028 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8029
8030 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8031 return rc;
8032}
8033
8034
8035/**
8036 * Saves the host state in the VMCS host-state.
8037 *
8038 * @returns VBox status code.
8039 * @param pVM Pointer to the VM.
8040 * @param pVCpu Pointer to the VMCPU.
8041 *
8042 * @remarks No-long-jump zone!!!
8043 */
8044VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8045{
8046 AssertPtr(pVM);
8047 AssertPtr(pVCpu);
8048
8049 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8050
8051 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8052 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8053 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8054 return hmR0VmxSaveHostState(pVM, pVCpu);
8055}
8056
8057
8058/**
8059 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8060 * loaded from these fields on every successful VM-entry.
8061 *
8062 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8063 * Sets up the VM-entry controls.
8064 * Sets up the appropriate VMX non-root function to execute guest code based on
8065 * the guest CPU mode.
8066 *
8067 * @returns VBox status code.
8068 * @param pVM Pointer to the VM.
8069 * @param pVCpu Pointer to the VMCPU.
8070 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8071 * out-of-sync. Make sure to update the required fields
8072 * before using them.
8073 *
8074 * @remarks No-long-jump zone!!!
8075 */
8076static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8077{
8078 AssertPtr(pVM);
8079 AssertPtr(pVCpu);
8080 AssertPtr(pMixedCtx);
8081 HMVMX_ASSERT_PREEMPT_SAFE();
8082
8083#ifdef LOG_ENABLED
8084 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8085 * probably not initialized yet? Anyway this will do for now.
8086 *
8087 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8088 * interface and disable ring-3 calls when thread-context hooks are not
8089 * available. */
8090 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8091 VMMR0LogFlushDisable(pVCpu);
8092#endif
8093
8094 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8095
8096 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8097
8098 /* Determine real-on-v86 mode. */
8099 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8100 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8101 && CPUMIsGuestInRealModeEx(pMixedCtx))
8102 {
8103 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8104 }
8105
8106 /*
8107 * Load the guest-state into the VMCS.
8108 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8109 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8110 */
8111 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8112 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8113
8114 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8115 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8116 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8117
8118 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8119 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8120 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8121
8122 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8123 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8124
8125 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8126 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8127
8128 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8129 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8130 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8131
8132 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8133 determine we don't have to swap EFER after all. */
8134 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8135 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8136
8137 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8138 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8139
8140 /*
8141 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8142 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8143 */
8144 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8145 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8146
8147 /* Clear any unused and reserved bits. */
8148 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8149
8150#ifdef LOG_ENABLED
8151 /* Only reenable log-flushing if the caller has it enabled. */
8152 if (!fCallerDisabledLogFlush)
8153 VMMR0LogFlushEnable(pVCpu);
8154#endif
8155
8156 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8157 return rc;
8158}
8159
8160
8161/**
8162 * Loads the state shared between the host and guest into the VMCS.
8163 *
8164 * @param pVM Pointer to the VM.
8165 * @param pVCpu Pointer to the VMCPU.
8166 * @param pCtx Pointer to the guest-CPU context.
8167 *
8168 * @remarks No-long-jump zone!!!
8169 */
8170static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8171{
8172 NOREF(pVM);
8173
8174 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8175 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8176
8177 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8178 {
8179 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8180 AssertRC(rc);
8181 }
8182
8183 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8184 {
8185 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8186 AssertRC(rc);
8187
8188 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8189 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8190 {
8191 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8192 AssertRC(rc);
8193 }
8194 }
8195
8196 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8197 {
8198#if HC_ARCH_BITS == 64
8199 if (pVM->hm.s.fAllow64BitGuests)
8200 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8201#endif
8202 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8203 }
8204
8205 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8206 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8207}
8208
8209
8210/**
8211 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8212 *
8213 * @param pVM Pointer to the VM.
8214 * @param pVCpu Pointer to the VMCPU.
8215 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8216 * out-of-sync. Make sure to update the required fields
8217 * before using them.
8218 */
8219DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8220{
8221 HMVMX_ASSERT_PREEMPT_SAFE();
8222
8223 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8224#ifdef HMVMX_SYNC_FULL_GUEST_STATE
8225 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8226#endif
8227
8228 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8229 {
8230 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8231 AssertRC(rc);
8232 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8233 }
8234 else if (HMCPU_CF_VALUE(pVCpu))
8235 {
8236 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8237 AssertRC(rc);
8238 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8239 }
8240
8241 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8242 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8243 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8244 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8245}
8246
8247
8248/**
8249 * Does the preparations before executing guest code in VT-x.
8250 *
8251 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8252 * recompiler. We must be cautious what we do here regarding committing
8253 * guest-state information into the VMCS assuming we assuredly execute the
8254 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8255 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8256 * so that the recompiler can (and should) use them when it resumes guest
8257 * execution. Otherwise such operations must be done when we can no longer
8258 * exit to ring-3.
8259 *
8260 * @returns Strict VBox status code.
8261 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8262 * have been disabled.
8263 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8264 * double-fault into the guest.
8265 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8266 *
8267 * @param pVM Pointer to the VM.
8268 * @param pVCpu Pointer to the VMCPU.
8269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8270 * out-of-sync. Make sure to update the required fields
8271 * before using them.
8272 * @param pVmxTransient Pointer to the VMX transient structure.
8273 */
8274static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8275{
8276 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8277
8278#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8279 PGMRZDynMapFlushAutoSet(pVCpu);
8280#endif
8281
8282 /* Check force flag actions that might require us to go back to ring-3. */
8283 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8284 if (rc != VINF_SUCCESS)
8285 return rc;
8286
8287#ifndef IEM_VERIFICATION_MODE_FULL
8288 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8289 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8290 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8291 {
8292 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8293 RTGCPHYS GCPhysApicBase;
8294 GCPhysApicBase = pMixedCtx->msrApicBase;
8295 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8296
8297 /* Unalias any existing mapping. */
8298 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8299 AssertRCReturn(rc, rc);
8300
8301 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8302 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8303 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8304 AssertRCReturn(rc, rc);
8305
8306 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8307 }
8308#endif /* !IEM_VERIFICATION_MODE_FULL */
8309
8310 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
8311 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8312
8313 /*
8314 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8315 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8316 */
8317 if (TRPMHasTrap(pVCpu))
8318 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8319 else if (!pVCpu->hm.s.Event.fPending)
8320 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8321
8322 /*
8323 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8324 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8325 */
8326 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8327 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8328 {
8329 Assert(rc == VINF_EM_RESET);
8330 return rc;
8331 }
8332
8333 /*
8334 * No longjmps to ring-3 from this point on!!!
8335 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8336 * This also disables flushing of the R0-logger instance (if any).
8337 */
8338 VMMRZCallRing3Disable(pVCpu);
8339
8340 /*
8341 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8342 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8343 *
8344 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8345 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8346 *
8347 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8348 * executing guest code.
8349 */
8350 pVmxTransient->uEflags = ASMIntDisableFlags();
8351 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8352 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8353 {
8354 hmR0VmxClearEventVmcs(pVCpu);
8355 ASMSetFlags(pVmxTransient->uEflags);
8356 VMMRZCallRing3Enable(pVCpu);
8357 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8358 return VINF_EM_RAW_TO_R3;
8359 }
8360
8361 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8362 {
8363 hmR0VmxClearEventVmcs(pVCpu);
8364 ASMSetFlags(pVmxTransient->uEflags);
8365 VMMRZCallRing3Enable(pVCpu);
8366 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8367 return VINF_EM_RAW_INTERRUPT;
8368 }
8369
8370 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8371 pVCpu->hm.s.Event.fPending = false;
8372
8373 return VINF_SUCCESS;
8374}
8375
8376
8377/**
8378 * Prepares to run guest code in VT-x and we've committed to doing so. This
8379 * means there is no backing out to ring-3 or anywhere else at this
8380 * point.
8381 *
8382 * @param pVM Pointer to the VM.
8383 * @param pVCpu Pointer to the VMCPU.
8384 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8385 * out-of-sync. Make sure to update the required fields
8386 * before using them.
8387 * @param pVmxTransient Pointer to the VMX transient structure.
8388 *
8389 * @remarks Called with preemption disabled.
8390 * @remarks No-long-jump zone!!!
8391 */
8392static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8393{
8394 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8395 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8396 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8397
8398 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8399 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8400
8401 /*
8402 * If we are injecting events to a real-on-v86 mode guest, we may have to update
8403 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8404 * Reload only the necessary state, the assertion will catch if other parts of the code
8405 * change.
8406 */
8407 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8408 {
8409 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8410 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8411 }
8412
8413#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8414 if (!CPUMIsGuestFPUStateActive(pVCpu))
8415 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8416 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8417#endif
8418
8419 if ( pVCpu->hm.s.fUseGuestFpu
8420 && !CPUMIsGuestFPUStateActive(pVCpu))
8421 {
8422 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8423 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8424 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8425 }
8426
8427 /*
8428 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8429 */
8430 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8431 && pVCpu->hm.s.vmx.cMsrs > 0)
8432 {
8433 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8434 }
8435
8436 /*
8437 * Load the host state bits as we may've been preempted (only happens when
8438 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8439 */
8440 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8441 {
8442 /* This ASSUMES that pfnStartVM has been set up already. */
8443 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8444 AssertRC(rc);
8445 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8446 }
8447 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8448
8449 /*
8450 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8451 */
8452 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8453 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8454 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8455
8456 /* Store status of the shared guest-host state at the time of VM-entry. */
8457#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8458 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8459 {
8460 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8461 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8462 }
8463 else
8464#endif
8465 {
8466 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8467 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8468 }
8469 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8470
8471 /*
8472 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8473 */
8474 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8475 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8476
8477 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8478 RTCPUID idCurrentCpu = pCpu->idCpu;
8479 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8480 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8481 {
8482 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8483 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8484 }
8485
8486 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8487 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8488 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8489 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8490
8491 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8492
8493 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8494 to start executing. */
8495
8496 /*
8497 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8498 */
8499 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8500 {
8501 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8502 {
8503 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8504 AssertRC(rc2);
8505 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8506 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8507 true /* fUpdateHostMsr */);
8508 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8509 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8510 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8511 }
8512 else
8513 {
8514 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8515 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8516 }
8517 }
8518
8519#ifdef VBOX_STRICT
8520 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8521 hmR0VmxCheckHostEferMsr(pVCpu);
8522 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8523#endif
8524#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8525 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8526 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8527 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8528#endif
8529}
8530
8531
8532/**
8533 * Performs some essential restoration of state after running guest code in
8534 * VT-x.
8535 *
8536 * @param pVM Pointer to the VM.
8537 * @param pVCpu Pointer to the VMCPU.
8538 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8539 * out-of-sync. Make sure to update the required fields
8540 * before using them.
8541 * @param pVmxTransient Pointer to the VMX transient structure.
8542 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8543 *
8544 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8545 *
8546 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8547 * unconditionally when it is safe to do so.
8548 */
8549static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8550{
8551 NOREF(pVM);
8552
8553 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8554
8555 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8556 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8557 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8558 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8559 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8560
8561 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8562 {
8563 /** @todo Find a way to fix hardcoding a guestimate. */
8564 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8565 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8566 }
8567
8568 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8569 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8570 Assert(!(ASMGetFlags() & X86_EFL_IF));
8571 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8572
8573#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8574 if (CPUMIsGuestFPUStateActive(pVCpu))
8575 {
8576 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8577 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8578 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8579 }
8580#endif
8581
8582#if HC_ARCH_BITS == 64
8583 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8584#endif
8585 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8586#ifdef VBOX_STRICT
8587 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8588#endif
8589 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8590 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8591
8592 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8593 uint32_t uExitReason;
8594 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8595 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8596 AssertRC(rc);
8597 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8598 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8599
8600 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8601 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8602 {
8603 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8604 pVmxTransient->fVMEntryFailed));
8605 return;
8606 }
8607
8608 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8609 {
8610 /* Update the guest interruptibility-state from the VMCS. */
8611 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8612#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8613 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8614 AssertRC(rc);
8615#endif
8616 /*
8617 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8618 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8619 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8620 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8621 */
8622 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8623 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8624 {
8625 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8626 AssertRC(rc);
8627 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8628 }
8629 }
8630}
8631
8632
8633/**
8634 * Runs the guest code using VT-x the normal way.
8635 *
8636 * @returns VBox status code.
8637 * @param pVM Pointer to the VM.
8638 * @param pVCpu Pointer to the VMCPU.
8639 * @param pCtx Pointer to the guest-CPU context.
8640 *
8641 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8642 */
8643static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8644{
8645 VMXTRANSIENT VmxTransient;
8646 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8647 int rc = VERR_INTERNAL_ERROR_5;
8648 uint32_t cLoops = 0;
8649
8650 for (;; cLoops++)
8651 {
8652 Assert(!HMR0SuspendPending());
8653 HMVMX_ASSERT_CPU_SAFE();
8654
8655 /* Preparatory work for running guest code, this may force us to return
8656 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8657 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8658 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8659 if (rc != VINF_SUCCESS)
8660 break;
8661
8662 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8663 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8664 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8665
8666 /* Restore any residual host-state and save any bits shared between host
8667 and guest into the guest-CPU state. Re-enables interrupts! */
8668 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8669
8670 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8671 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8672 {
8673 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8674 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8675 return rc;
8676 }
8677
8678 /* Handle the VM-exit. */
8679 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8680 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8681 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8682 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8683 HMVMX_START_EXIT_DISPATCH_PROF();
8684#ifdef HMVMX_USE_FUNCTION_TABLE
8685 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8686#else
8687 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8688#endif
8689 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8690 if (rc != VINF_SUCCESS)
8691 break;
8692 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8693 {
8694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8695 rc = VINF_EM_RAW_INTERRUPT;
8696 break;
8697 }
8698 }
8699
8700 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8701 return rc;
8702}
8703
8704
8705/**
8706 * Single steps guest code using VT-x.
8707 *
8708 * @returns VBox status code.
8709 * @param pVM Pointer to the VM.
8710 * @param pVCpu Pointer to the VMCPU.
8711 * @param pCtx Pointer to the guest-CPU context.
8712 *
8713 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8714 */
8715static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8716{
8717 VMXTRANSIENT VmxTransient;
8718 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8719 int rc = VERR_INTERNAL_ERROR_5;
8720 uint32_t cLoops = 0;
8721 uint16_t uCsStart = pCtx->cs.Sel;
8722 uint64_t uRipStart = pCtx->rip;
8723
8724 for (;; cLoops++)
8725 {
8726 Assert(!HMR0SuspendPending());
8727 HMVMX_ASSERT_CPU_SAFE();
8728
8729 /* Preparatory work for running guest code, this may force us to return
8730 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8731 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8732 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8733 if (rc != VINF_SUCCESS)
8734 break;
8735
8736 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8737 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8738 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8739
8740 /* Restore any residual host-state and save any bits shared between host
8741 and guest into the guest-CPU state. Re-enables interrupts! */
8742 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8743
8744 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8745 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8746 {
8747 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8748 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8749 return rc;
8750 }
8751
8752 /* Handle the VM-exit. */
8753 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8754 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8755 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8756 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8757 HMVMX_START_EXIT_DISPATCH_PROF();
8758#ifdef HMVMX_USE_FUNCTION_TABLE
8759 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8760#else
8761 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8762#endif
8763 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8764 if (rc != VINF_SUCCESS)
8765 break;
8766 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8767 {
8768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8769 rc = VINF_EM_RAW_INTERRUPT;
8770 break;
8771 }
8772
8773 /*
8774 * Did the RIP change, if so, consider it a single step.
8775 * Otherwise, make sure one of the TFs gets set.
8776 */
8777 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8778 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8779 AssertRCReturn(rc2, rc2);
8780 if ( pCtx->rip != uRipStart
8781 || pCtx->cs.Sel != uCsStart)
8782 {
8783 rc = VINF_EM_DBG_STEPPED;
8784 break;
8785 }
8786 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8787 }
8788
8789 /*
8790 * Clear the X86_EFL_TF if necessary.
8791 */
8792 if (pVCpu->hm.s.fClearTrapFlag)
8793 {
8794 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8795 AssertRCReturn(rc2, rc2);
8796 pVCpu->hm.s.fClearTrapFlag = false;
8797 pCtx->eflags.Bits.u1TF = 0;
8798 }
8799 /** @todo there seems to be issues with the resume flag when the monitor trap
8800 * flag is pending without being used. Seen early in bios init when
8801 * accessing APIC page in protected mode. */
8802
8803 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8804 return rc;
8805}
8806
8807
8808/**
8809 * Runs the guest code using VT-x.
8810 *
8811 * @returns VBox status code.
8812 * @param pVM Pointer to the VM.
8813 * @param pVCpu Pointer to the VMCPU.
8814 * @param pCtx Pointer to the guest-CPU context.
8815 */
8816VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8817{
8818 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8819 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8820 HMVMX_ASSERT_PREEMPT_SAFE();
8821
8822 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8823
8824 int rc;
8825 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8826 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8827 else
8828 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8829
8830 if (rc == VERR_EM_INTERPRETER)
8831 rc = VINF_EM_RAW_EMULATE_INSTR;
8832 else if (rc == VINF_EM_RESET)
8833 rc = VINF_EM_TRIPLE_FAULT;
8834
8835 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8836 if (RT_FAILURE(rc2))
8837 {
8838 pVCpu->hm.s.u32HMError = rc;
8839 rc = rc2;
8840 }
8841 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8842 return rc;
8843}
8844
8845
8846#ifndef HMVMX_USE_FUNCTION_TABLE
8847DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8848{
8849#ifdef DEBUG_ramshankar
8850# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8851# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8852#endif
8853 int rc;
8854 switch (rcReason)
8855 {
8856 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8857 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8858 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8859 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8860 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8861 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8862 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8863 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8864 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8865 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8866 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8867 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8868 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8869 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8870 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8871 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8872 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8873 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8874 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8875 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8876 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8877 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8878 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8879 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8880 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8881 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8882 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8883 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8884 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8885 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8886 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8887 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8888 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8889
8890 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8891 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8892 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8893 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8894 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8895 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8896 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8897 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8898 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8899
8900 case VMX_EXIT_VMCALL:
8901 case VMX_EXIT_VMCLEAR:
8902 case VMX_EXIT_VMLAUNCH:
8903 case VMX_EXIT_VMPTRLD:
8904 case VMX_EXIT_VMPTRST:
8905 case VMX_EXIT_VMREAD:
8906 case VMX_EXIT_VMRESUME:
8907 case VMX_EXIT_VMWRITE:
8908 case VMX_EXIT_VMXOFF:
8909 case VMX_EXIT_VMXON:
8910 case VMX_EXIT_INVEPT:
8911 case VMX_EXIT_INVVPID:
8912 case VMX_EXIT_VMFUNC:
8913 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8914 break;
8915 default:
8916 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8917 break;
8918 }
8919 return rc;
8920}
8921#endif
8922
8923#ifdef DEBUG
8924/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8925# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8926 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8927
8928# define HMVMX_ASSERT_PREEMPT_CPUID() \
8929 do \
8930 { \
8931 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8932 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8933 } while (0)
8934
8935# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8936 do { \
8937 AssertPtr(pVCpu); \
8938 AssertPtr(pMixedCtx); \
8939 AssertPtr(pVmxTransient); \
8940 Assert(pVmxTransient->fVMEntryFailed == false); \
8941 Assert(ASMIntAreEnabled()); \
8942 HMVMX_ASSERT_PREEMPT_SAFE(); \
8943 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8944 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8945 HMVMX_ASSERT_PREEMPT_SAFE(); \
8946 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8947 HMVMX_ASSERT_PREEMPT_CPUID(); \
8948 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8949 } while (0)
8950
8951# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8952 do { \
8953 Log4Func(("\n")); \
8954 } while (0)
8955#else /* Release builds */
8956# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8957 do { \
8958 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8959 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8960 } while (0)
8961# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8962#endif
8963
8964
8965/**
8966 * Advances the guest RIP after reading it from the VMCS.
8967 *
8968 * @returns VBox status code.
8969 * @param pVCpu Pointer to the VMCPU.
8970 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8971 * out-of-sync. Make sure to update the required fields
8972 * before using them.
8973 * @param pVmxTransient Pointer to the VMX transient structure.
8974 *
8975 * @remarks No-long-jump zone!!!
8976 */
8977DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8978{
8979 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8980 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8981 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8982 AssertRCReturn(rc, rc);
8983
8984 pMixedCtx->rip += pVmxTransient->cbInstr;
8985 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8986
8987 /*
8988 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8989 * pending debug exception field as it takes care of priority of events.
8990 *
8991 * See Intel spec. 32.2.1 "Debug Exceptions".
8992 */
8993 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8994
8995 return rc;
8996}
8997
8998
8999/**
9000 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9001 * and update error record fields accordingly.
9002 *
9003 * @return VMX_IGS_* return codes.
9004 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9005 * wrong with the guest state.
9006 *
9007 * @param pVM Pointer to the VM.
9008 * @param pVCpu Pointer to the VMCPU.
9009 * @param pCtx Pointer to the guest-CPU state.
9010 *
9011 * @remarks This function assumes our cache of the VMCS controls
9012 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9013 */
9014static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9015{
9016#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9017#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9018 uError = (err); \
9019 break; \
9020 } else do { } while (0)
9021
9022 int rc;
9023 uint32_t uError = VMX_IGS_ERROR;
9024 uint32_t u32Val;
9025 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9026
9027 do
9028 {
9029 /*
9030 * CR0.
9031 */
9032 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9033 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9034 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9035 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9036 if (fUnrestrictedGuest)
9037 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9038
9039 uint32_t u32GuestCR0;
9040 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9041 AssertRCBreak(rc);
9042 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9043 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9044 if ( !fUnrestrictedGuest
9045 && (u32GuestCR0 & X86_CR0_PG)
9046 && !(u32GuestCR0 & X86_CR0_PE))
9047 {
9048 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9049 }
9050
9051 /*
9052 * CR4.
9053 */
9054 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9055 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9056
9057 uint32_t u32GuestCR4;
9058 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9059 AssertRCBreak(rc);
9060 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9061 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9062
9063 /*
9064 * IA32_DEBUGCTL MSR.
9065 */
9066 uint64_t u64Val;
9067 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9068 AssertRCBreak(rc);
9069 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9070 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9071 {
9072 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9073 }
9074 uint64_t u64DebugCtlMsr = u64Val;
9075
9076#ifdef VBOX_STRICT
9077 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9078 AssertRCBreak(rc);
9079 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9080#endif
9081 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9082
9083 /*
9084 * RIP and RFLAGS.
9085 */
9086 uint32_t u32Eflags;
9087#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9088 if (HMVMX_IS_64BIT_HOST_MODE())
9089 {
9090 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9091 AssertRCBreak(rc);
9092 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9093 if ( !fLongModeGuest
9094 || !pCtx->cs.Attr.n.u1Long)
9095 {
9096 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9097 }
9098 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9099 * must be identical if the "IA32e mode guest" VM-entry control is 1
9100 * and CS.L is 1. No check applies if the CPU supports 64
9101 * linear-address bits. */
9102
9103 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9104 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9105 AssertRCBreak(rc);
9106 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9107 VMX_IGS_RFLAGS_RESERVED);
9108 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9109 u32Eflags = u64Val;
9110 }
9111 else
9112#endif
9113 {
9114 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9115 AssertRCBreak(rc);
9116 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9117 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9118 }
9119
9120 if ( fLongModeGuest
9121 || ( fUnrestrictedGuest
9122 && !(u32GuestCR0 & X86_CR0_PE)))
9123 {
9124 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9125 }
9126
9127 uint32_t u32EntryInfo;
9128 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9129 AssertRCBreak(rc);
9130 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9131 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9132 {
9133 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9134 }
9135
9136 /*
9137 * 64-bit checks.
9138 */
9139#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9140 if (HMVMX_IS_64BIT_HOST_MODE())
9141 {
9142 if ( fLongModeGuest
9143 && !fUnrestrictedGuest)
9144 {
9145 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9146 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9147 }
9148
9149 if ( !fLongModeGuest
9150 && (u32GuestCR4 & X86_CR4_PCIDE))
9151 {
9152 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9153 }
9154
9155 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9156 * 51:32 beyond the processor's physical-address width are 0. */
9157
9158 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9159 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9160 {
9161 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9162 }
9163
9164 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9165 AssertRCBreak(rc);
9166 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9167
9168 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9169 AssertRCBreak(rc);
9170 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9171 }
9172#endif
9173
9174 /*
9175 * PERF_GLOBAL MSR.
9176 */
9177 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9178 {
9179 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9180 AssertRCBreak(rc);
9181 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9182 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9183 }
9184
9185 /*
9186 * PAT MSR.
9187 */
9188 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9189 {
9190 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9191 AssertRCBreak(rc);
9192 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9193 for (unsigned i = 0; i < 8; i++)
9194 {
9195 uint8_t u8Val = (u64Val & 0x7);
9196 if ( u8Val != 0 /* UC */
9197 || u8Val != 1 /* WC */
9198 || u8Val != 4 /* WT */
9199 || u8Val != 5 /* WP */
9200 || u8Val != 6 /* WB */
9201 || u8Val != 7 /* UC- */)
9202 {
9203 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9204 }
9205 u64Val >>= 3;
9206 }
9207 }
9208
9209 /*
9210 * EFER MSR.
9211 */
9212 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9213 {
9214 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9215 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9216 AssertRCBreak(rc);
9217 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9218 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9219 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9220 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9221 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9222 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9223 VMX_IGS_EFER_LMA_PG_MISMATCH);
9224 }
9225
9226 /*
9227 * Segment registers.
9228 */
9229 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9230 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9231 if (!(u32Eflags & X86_EFL_VM))
9232 {
9233 /* CS */
9234 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9235 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9236 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9237 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9238 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9239 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9240 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9241 /* CS cannot be loaded with NULL in protected mode. */
9242 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9243 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9244 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9245 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9246 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9247 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9248 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9249 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9250 else
9251 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9252
9253 /* SS */
9254 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9255 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9256 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9257 if ( !(pCtx->cr0 & X86_CR0_PE)
9258 || pCtx->cs.Attr.n.u4Type == 3)
9259 {
9260 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9261 }
9262 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9263 {
9264 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9265 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9266 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9267 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9268 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9269 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9270 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9271 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9272 }
9273
9274 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9275 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9276 {
9277 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9278 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9279 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9280 || pCtx->ds.Attr.n.u4Type > 11
9281 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9282 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9283 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9284 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9285 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9286 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9287 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9288 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9289 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9290 }
9291 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9292 {
9293 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9294 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9295 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9296 || pCtx->es.Attr.n.u4Type > 11
9297 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9298 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9299 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9300 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9301 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9302 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9303 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9304 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9305 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9306 }
9307 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9308 {
9309 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9310 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9311 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9312 || pCtx->fs.Attr.n.u4Type > 11
9313 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9314 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9315 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9316 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9317 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9318 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9319 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9320 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9321 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9322 }
9323 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9324 {
9325 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9326 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9327 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9328 || pCtx->gs.Attr.n.u4Type > 11
9329 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9330 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9331 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9332 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9333 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9334 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9335 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9336 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9337 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9338 }
9339 /* 64-bit capable CPUs. */
9340#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9341 if (HMVMX_IS_64BIT_HOST_MODE())
9342 {
9343 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9344 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9345 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9346 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9347 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9348 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9349 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9350 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9351 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9352 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9353 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9354 }
9355#endif
9356 }
9357 else
9358 {
9359 /* V86 mode checks. */
9360 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9361 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9362 {
9363 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9364 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9365 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9366 }
9367 else
9368 {
9369 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9370 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9371 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9372 }
9373
9374 /* CS */
9375 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9376 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9377 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9378 /* SS */
9379 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9380 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9381 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9382 /* DS */
9383 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9384 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9385 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9386 /* ES */
9387 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9388 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9389 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9390 /* FS */
9391 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9392 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9393 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9394 /* GS */
9395 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9396 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9397 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9398 /* 64-bit capable CPUs. */
9399#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9400 if (HMVMX_IS_64BIT_HOST_MODE())
9401 {
9402 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9403 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9404 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9405 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9406 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9407 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9408 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9409 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9410 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9411 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9412 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9413 }
9414#endif
9415 }
9416
9417 /*
9418 * TR.
9419 */
9420 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9421 /* 64-bit capable CPUs. */
9422#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9423 if (HMVMX_IS_64BIT_HOST_MODE())
9424 {
9425 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9426 }
9427#endif
9428 if (fLongModeGuest)
9429 {
9430 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9431 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9432 }
9433 else
9434 {
9435 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9436 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9437 VMX_IGS_TR_ATTR_TYPE_INVALID);
9438 }
9439 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9440 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9441 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9442 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9443 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9444 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9445 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9446 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9447
9448 /*
9449 * GDTR and IDTR.
9450 */
9451#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9452 if (HMVMX_IS_64BIT_HOST_MODE())
9453 {
9454 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9455 AssertRCBreak(rc);
9456 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9457
9458 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9459 AssertRCBreak(rc);
9460 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9461 }
9462#endif
9463
9464 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9465 AssertRCBreak(rc);
9466 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9467
9468 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9469 AssertRCBreak(rc);
9470 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9471
9472 /*
9473 * Guest Non-Register State.
9474 */
9475 /* Activity State. */
9476 uint32_t u32ActivityState;
9477 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9478 AssertRCBreak(rc);
9479 HMVMX_CHECK_BREAK( !u32ActivityState
9480 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9481 VMX_IGS_ACTIVITY_STATE_INVALID);
9482 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9483 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9484 uint32_t u32IntrState;
9485 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9486 AssertRCBreak(rc);
9487 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9488 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9489 {
9490 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9491 }
9492
9493 /** @todo Activity state and injecting interrupts. Left as a todo since we
9494 * currently don't use activity states but ACTIVE. */
9495
9496 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9497 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9498
9499 /* Guest interruptibility-state. */
9500 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9501 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9502 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9503 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9504 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9505 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9506 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9507 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9508 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9509 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9510 {
9511 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9512 {
9513 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9514 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9515 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9516 }
9517 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9518 {
9519 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9520 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9521 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9522 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9523 }
9524 }
9525 /** @todo Assumes the processor is not in SMM. */
9526 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9527 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9528 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9529 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9530 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9531 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9532 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9533 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9534 {
9535 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9536 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9537 }
9538
9539 /* Pending debug exceptions. */
9540 if (HMVMX_IS_64BIT_HOST_MODE())
9541 {
9542 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9543 AssertRCBreak(rc);
9544 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9545 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9546 u32Val = u64Val; /* For pending debug exceptions checks below. */
9547 }
9548 else
9549 {
9550 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9551 AssertRCBreak(rc);
9552 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9553 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9554 }
9555
9556 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9557 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9558 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9559 {
9560 if ( (u32Eflags & X86_EFL_TF)
9561 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9562 {
9563 /* Bit 14 is PendingDebug.BS. */
9564 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9565 }
9566 if ( !(u32Eflags & X86_EFL_TF)
9567 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9568 {
9569 /* Bit 14 is PendingDebug.BS. */
9570 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9571 }
9572 }
9573
9574 /* VMCS link pointer. */
9575 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9576 AssertRCBreak(rc);
9577 if (u64Val != UINT64_C(0xffffffffffffffff))
9578 {
9579 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9580 /** @todo Bits beyond the processor's physical-address width MBZ. */
9581 /** @todo 32-bit located in memory referenced by value of this field (as a
9582 * physical address) must contain the processor's VMCS revision ID. */
9583 /** @todo SMM checks. */
9584 }
9585
9586 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9587 * not using Nested Paging? */
9588 if ( pVM->hm.s.fNestedPaging
9589 && !fLongModeGuest
9590 && CPUMIsGuestInPAEModeEx(pCtx))
9591 {
9592 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9593 AssertRCBreak(rc);
9594 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9595
9596 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9597 AssertRCBreak(rc);
9598 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9599
9600 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9601 AssertRCBreak(rc);
9602 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9603
9604 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9605 AssertRCBreak(rc);
9606 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9607 }
9608
9609 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9610 if (uError == VMX_IGS_ERROR)
9611 uError = VMX_IGS_REASON_NOT_FOUND;
9612 } while (0);
9613
9614 pVCpu->hm.s.u32HMError = uError;
9615 return uError;
9616
9617#undef HMVMX_ERROR_BREAK
9618#undef HMVMX_CHECK_BREAK
9619}
9620
9621/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9622/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9623/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9624
9625/** @name VM-exit handlers.
9626 * @{
9627 */
9628
9629/**
9630 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9631 */
9632HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9633{
9634 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9636 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9637 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9638 return VINF_SUCCESS;
9639 return VINF_EM_RAW_INTERRUPT;
9640}
9641
9642
9643/**
9644 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9645 */
9646HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9647{
9648 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9649 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9650
9651 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9652 AssertRCReturn(rc, rc);
9653
9654 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9655 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9656 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9657 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9658
9659 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9660 {
9661 /*
9662 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9663 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9664 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9665 *
9666 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9667 */
9668 VMXDispatchHostNmi();
9669 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9670 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9671 return VINF_SUCCESS;
9672 }
9673
9674 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9675 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9676 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9677 {
9678 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9679 return VINF_SUCCESS;
9680 }
9681 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9682 {
9683 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9684 return rc;
9685 }
9686
9687 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9688 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9689 switch (uIntType)
9690 {
9691 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9692 Assert(uVector == X86_XCPT_DB);
9693 /* no break */
9694 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9695 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9696 /* no break */
9697 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9698 {
9699 switch (uVector)
9700 {
9701 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9702 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9703 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9704 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9705 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9706 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9707#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9708 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9709 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9710 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9711 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9712 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9713 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9714 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9715 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9716 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9717 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9718 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9719 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9720#endif
9721 default:
9722 {
9723 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9724 AssertRCReturn(rc, rc);
9725
9726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9727 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9728 {
9729 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9730 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9731 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9732
9733 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9734 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9735 AssertRCReturn(rc, rc);
9736 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9737 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9738 0 /* GCPtrFaultAddress */);
9739 AssertRCReturn(rc, rc);
9740 }
9741 else
9742 {
9743 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9744 pVCpu->hm.s.u32HMError = uVector;
9745 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9746 }
9747 break;
9748 }
9749 }
9750 break;
9751 }
9752
9753 default:
9754 {
9755 pVCpu->hm.s.u32HMError = uExitIntInfo;
9756 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9757 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9758 break;
9759 }
9760 }
9761 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9762 return rc;
9763}
9764
9765
9766/**
9767 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9768 */
9769HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9770{
9771 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9772
9773 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9774 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9775
9776 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9778 return VINF_SUCCESS;
9779}
9780
9781
9782/**
9783 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9784 */
9785HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9786{
9787 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9788 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9789 HMVMX_RETURN_UNEXPECTED_EXIT();
9790}
9791
9792
9793/**
9794 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9795 */
9796HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9797{
9798 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9800 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9801}
9802
9803
9804/**
9805 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9806 */
9807HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9808{
9809 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9811 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9812}
9813
9814
9815/**
9816 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9817 */
9818HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819{
9820 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9821 PVM pVM = pVCpu->CTX_SUFF(pVM);
9822 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9823 if (RT_LIKELY(rc == VINF_SUCCESS))
9824 {
9825 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9826 Assert(pVmxTransient->cbInstr == 2);
9827 }
9828 else
9829 {
9830 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9831 rc = VERR_EM_INTERPRETER;
9832 }
9833 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9834 return rc;
9835}
9836
9837
9838/**
9839 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9840 */
9841HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9842{
9843 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9844 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9845 AssertRCReturn(rc, rc);
9846
9847 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9848 return VINF_EM_RAW_EMULATE_INSTR;
9849
9850 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9851 HMVMX_RETURN_UNEXPECTED_EXIT();
9852}
9853
9854
9855/**
9856 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9857 */
9858HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9859{
9860 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9861 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9862 AssertRCReturn(rc, rc);
9863
9864 PVM pVM = pVCpu->CTX_SUFF(pVM);
9865 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9866 if (RT_LIKELY(rc == VINF_SUCCESS))
9867 {
9868 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9869 Assert(pVmxTransient->cbInstr == 2);
9870 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9871 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9872 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9873 }
9874 else
9875 {
9876 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9877 rc = VERR_EM_INTERPRETER;
9878 }
9879 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9880 return rc;
9881}
9882
9883
9884/**
9885 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9886 */
9887HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9888{
9889 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9890 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9891 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9892 AssertRCReturn(rc, rc);
9893
9894 PVM pVM = pVCpu->CTX_SUFF(pVM);
9895 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9896 if (RT_LIKELY(rc == VINF_SUCCESS))
9897 {
9898 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9899 Assert(pVmxTransient->cbInstr == 3);
9900 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9901 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9902 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9903 }
9904 else
9905 {
9906 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9907 rc = VERR_EM_INTERPRETER;
9908 }
9909 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9910 return rc;
9911}
9912
9913
9914/**
9915 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9916 */
9917HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9918{
9919 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9920 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9921 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9922 AssertRCReturn(rc, rc);
9923
9924 PVM pVM = pVCpu->CTX_SUFF(pVM);
9925 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9926 if (RT_LIKELY(rc == VINF_SUCCESS))
9927 {
9928 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9929 Assert(pVmxTransient->cbInstr == 2);
9930 }
9931 else
9932 {
9933 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9934 rc = VERR_EM_INTERPRETER;
9935 }
9936 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9937 return rc;
9938}
9939
9940
9941/**
9942 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9943 */
9944HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9945{
9946 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9947 PVM pVM = pVCpu->CTX_SUFF(pVM);
9948 Assert(!pVM->hm.s.fNestedPaging);
9949
9950 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9951 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9952 AssertRCReturn(rc, rc);
9953
9954 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9955 rc = VBOXSTRICTRC_VAL(rc2);
9956 if (RT_LIKELY(rc == VINF_SUCCESS))
9957 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9958 else
9959 {
9960 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9961 pVmxTransient->uExitQualification, rc));
9962 }
9963 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9964 return rc;
9965}
9966
9967
9968/**
9969 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9970 */
9971HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9972{
9973 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9974 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9975 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9976 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9977 AssertRCReturn(rc, rc);
9978
9979 PVM pVM = pVCpu->CTX_SUFF(pVM);
9980 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9981 if (RT_LIKELY(rc == VINF_SUCCESS))
9982 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9983 else
9984 {
9985 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9986 rc = VERR_EM_INTERPRETER;
9987 }
9988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9989 return rc;
9990}
9991
9992
9993/**
9994 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9995 */
9996HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9997{
9998 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9999 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10000 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10001 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10002 AssertRCReturn(rc, rc);
10003
10004 PVM pVM = pVCpu->CTX_SUFF(pVM);
10005 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10006 rc = VBOXSTRICTRC_VAL(rc2);
10007 if (RT_LIKELY( rc == VINF_SUCCESS
10008 || rc == VINF_EM_HALT))
10009 {
10010 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10011 AssertRCReturn(rc3, rc3);
10012
10013 if ( rc == VINF_EM_HALT
10014 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10015 {
10016 rc = VINF_SUCCESS;
10017 }
10018 }
10019 else
10020 {
10021 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10022 rc = VERR_EM_INTERPRETER;
10023 }
10024 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10025 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10026 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10027 return rc;
10028}
10029
10030
10031/**
10032 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10033 */
10034HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10035{
10036 /*
10037 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10038 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10039 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10040 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10041 */
10042 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10043 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10044 HMVMX_RETURN_UNEXPECTED_EXIT();
10045}
10046
10047
10048/**
10049 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10050 */
10051HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10052{
10053 /*
10054 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10055 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10056 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10057 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10058 */
10059 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10060 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10061 HMVMX_RETURN_UNEXPECTED_EXIT();
10062}
10063
10064
10065/**
10066 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10067 */
10068HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10069{
10070 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10071 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10072 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10073 HMVMX_RETURN_UNEXPECTED_EXIT();
10074}
10075
10076
10077/**
10078 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10079 */
10080HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10081{
10082 /*
10083 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10084 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10085 * See Intel spec. 25.3 "Other Causes of VM-exits".
10086 */
10087 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10088 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10089 HMVMX_RETURN_UNEXPECTED_EXIT();
10090}
10091
10092
10093/**
10094 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10095 * VM-exit.
10096 */
10097HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10098{
10099 /*
10100 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10101 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10102 *
10103 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10104 * See Intel spec. "23.8 Restrictions on VMX operation".
10105 */
10106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10107 return VINF_SUCCESS;
10108}
10109
10110
10111/**
10112 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10113 * VM-exit.
10114 */
10115HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10116{
10117 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10118 return VINF_EM_RESET;
10119}
10120
10121
10122/**
10123 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10124 */
10125HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10126{
10127 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10128 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10129 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10130 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10131 AssertRCReturn(rc, rc);
10132
10133 pMixedCtx->rip++;
10134 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10135 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10136 rc = VINF_SUCCESS;
10137 else
10138 rc = VINF_EM_HALT;
10139
10140 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10141 return rc;
10142}
10143
10144
10145/**
10146 * VM-exit handler for instructions that result in a #UD exception delivered to
10147 * the guest.
10148 */
10149HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10150{
10151 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10152 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10153 return VINF_SUCCESS;
10154}
10155
10156
10157/**
10158 * VM-exit handler for expiry of the VMX preemption timer.
10159 */
10160HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10161{
10162 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10163
10164 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10165 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10166
10167 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10168 PVM pVM = pVCpu->CTX_SUFF(pVM);
10169 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10171 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10172}
10173
10174
10175/**
10176 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10177 */
10178HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10179{
10180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10181
10182 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10183 /** @todo check if XSETBV is supported by the recompiler. */
10184 return VERR_EM_INTERPRETER;
10185}
10186
10187
10188/**
10189 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10190 */
10191HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10192{
10193 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10194
10195 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10196 /** @todo implement EMInterpretInvpcid() */
10197 return VERR_EM_INTERPRETER;
10198}
10199
10200
10201/**
10202 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10203 * Error VM-exit.
10204 */
10205HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10206{
10207 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10208 AssertRCReturn(rc, rc);
10209
10210 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10211 AssertRCReturn(rc, rc);
10212
10213 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10214 NOREF(uInvalidReason);
10215
10216#ifdef VBOX_STRICT
10217 uint32_t uIntrState;
10218 HMVMXHCUINTREG uHCReg;
10219 uint64_t u64Val;
10220 uint32_t u32Val;
10221
10222 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10223 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10224 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10225 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10226 AssertRCReturn(rc, rc);
10227
10228 Log4(("uInvalidReason %u\n", uInvalidReason));
10229 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10230 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10231 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10232 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10233
10234 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10235 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10236 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10237 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10238 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10239 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10240 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10241 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10242 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10243 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10244 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10245 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10246#else
10247 NOREF(pVmxTransient);
10248#endif
10249
10250 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10251 return VERR_VMX_INVALID_GUEST_STATE;
10252}
10253
10254
10255/**
10256 * VM-exit handler for VM-entry failure due to an MSR-load
10257 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10258 */
10259HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10260{
10261 NOREF(pVmxTransient);
10262 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10263 HMVMX_RETURN_UNEXPECTED_EXIT();
10264}
10265
10266
10267/**
10268 * VM-exit handler for VM-entry failure due to a machine-check event
10269 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10270 */
10271HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10272{
10273 NOREF(pVmxTransient);
10274 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10275 HMVMX_RETURN_UNEXPECTED_EXIT();
10276}
10277
10278
10279/**
10280 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10281 * theory.
10282 */
10283HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10284{
10285 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10286 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10287 return VERR_VMX_UNDEFINED_EXIT_CODE;
10288}
10289
10290
10291/**
10292 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10293 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10294 * Conditional VM-exit.
10295 */
10296HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10297{
10298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10299
10300 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10301 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10302 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10303 return VERR_EM_INTERPRETER;
10304 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10305 HMVMX_RETURN_UNEXPECTED_EXIT();
10306}
10307
10308
10309/**
10310 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10311 */
10312HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10313{
10314 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10315
10316 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10318 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10319 return VERR_EM_INTERPRETER;
10320 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10321 HMVMX_RETURN_UNEXPECTED_EXIT();
10322}
10323
10324
10325/**
10326 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10327 */
10328HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10329{
10330 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10331
10332 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10333 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10334 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10335 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10336 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10337 {
10338 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10339 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10340 }
10341 AssertRCReturn(rc, rc);
10342 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10343
10344#ifdef VBOX_STRICT
10345 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10346 {
10347 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10348 && pMixedCtx->ecx != MSR_K6_EFER)
10349 {
10350 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10351 HMVMX_RETURN_UNEXPECTED_EXIT();
10352 }
10353# if HC_ARCH_BITS == 64
10354 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10355 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10356 {
10357 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10358 HMVMX_RETURN_UNEXPECTED_EXIT();
10359 }
10360# endif
10361 }
10362#endif
10363
10364 PVM pVM = pVCpu->CTX_SUFF(pVM);
10365 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10366 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10367 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10369
10370 if (RT_LIKELY(rc == VINF_SUCCESS))
10371 {
10372 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10373 Assert(pVmxTransient->cbInstr == 2);
10374 }
10375 return rc;
10376}
10377
10378
10379/**
10380 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10381 */
10382HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10383{
10384 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10385 PVM pVM = pVCpu->CTX_SUFF(pVM);
10386 int rc = VINF_SUCCESS;
10387
10388 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10389 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10390 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10391 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10392 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10393 {
10394 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10395 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10396 }
10397 AssertRCReturn(rc, rc);
10398 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10399
10400 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10401 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10402 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10403
10404 if (RT_LIKELY(rc == VINF_SUCCESS))
10405 {
10406 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10407
10408 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10409 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10410 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10411 {
10412 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10413 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10414 EMInterpretWrmsr() changes it. */
10415 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10416 }
10417 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10418 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10419 else if (pMixedCtx->ecx == MSR_K6_EFER)
10420 {
10421 /*
10422 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10423 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10424 * the other bits as well, SCE and NXE. See @bugref{7368}.
10425 */
10426 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10427 }
10428
10429 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10430 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10431 {
10432 switch (pMixedCtx->ecx)
10433 {
10434 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10435 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10436 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10437 case MSR_K8_FS_BASE: /* no break */
10438 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10439 case MSR_K6_EFER: /* already handled above */ break;
10440 default:
10441 {
10442 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10443 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10444#if HC_ARCH_BITS == 64
10445 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10446 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10447#endif
10448 break;
10449 }
10450 }
10451 }
10452#ifdef VBOX_STRICT
10453 else
10454 {
10455 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10456 switch (pMixedCtx->ecx)
10457 {
10458 case MSR_IA32_SYSENTER_CS:
10459 case MSR_IA32_SYSENTER_EIP:
10460 case MSR_IA32_SYSENTER_ESP:
10461 case MSR_K8_FS_BASE:
10462 case MSR_K8_GS_BASE:
10463 {
10464 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10465 HMVMX_RETURN_UNEXPECTED_EXIT();
10466 }
10467
10468 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10469 default:
10470 {
10471 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10472 {
10473#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10474 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10475 if ( HMVMX_IS_64BIT_HOST_MODE()
10476 && pMixedCtx->ecx == MSR_K6_EFER)
10477 {
10478 break;
10479 }
10480#endif
10481 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10482 pMixedCtx->ecx));
10483 HMVMX_RETURN_UNEXPECTED_EXIT();
10484 }
10485
10486#if HC_ARCH_BITS == 64
10487 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10488 {
10489 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10490 HMVMX_RETURN_UNEXPECTED_EXIT();
10491 }
10492#endif
10493 break;
10494 }
10495 }
10496 }
10497#endif /* VBOX_STRICT */
10498 }
10499 return rc;
10500}
10501
10502
10503/**
10504 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10505 */
10506HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10507{
10508 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10509
10510 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10511 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10512 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10513 return VERR_EM_INTERPRETER;
10514 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10515 HMVMX_RETURN_UNEXPECTED_EXIT();
10516}
10517
10518
10519/**
10520 * VM-exit handler for when the TPR value is lowered below the specified
10521 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10522 */
10523HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10524{
10525 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10526 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10527
10528 /*
10529 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10530 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10531 * resume guest execution.
10532 */
10533 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10534 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10535 return VINF_SUCCESS;
10536}
10537
10538
10539/**
10540 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10541 * VM-exit.
10542 *
10543 * @retval VINF_SUCCESS when guest execution can continue.
10544 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10545 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10546 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10547 * recompiler.
10548 */
10549HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10550{
10551 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10552 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10553 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10554 AssertRCReturn(rc, rc);
10555
10556 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10557 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10558 PVM pVM = pVCpu->CTX_SUFF(pVM);
10559 switch (uAccessType)
10560 {
10561 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10562 {
10563#if 0
10564 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10565 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10566#else
10567 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10568 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10569 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10570#endif
10571 AssertRCReturn(rc, rc);
10572
10573 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10574 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10575 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10576 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10577
10578 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10579 {
10580 case 0: /* CR0 */
10581 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10582 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10583 break;
10584 case 2: /* CR2 */
10585 /* Nothing to do here, CR2 it's not part of the VMCS. */
10586 break;
10587 case 3: /* CR3 */
10588 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10589 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10590 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10591 break;
10592 case 4: /* CR4 */
10593 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10594 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10595 break;
10596 case 8: /* CR8 */
10597 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10598 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10599 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10600 break;
10601 default:
10602 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10603 break;
10604 }
10605
10606 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10607 break;
10608 }
10609
10610 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10611 {
10612 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10613 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10614 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10615 AssertRCReturn(rc, rc);
10616 Assert( !pVM->hm.s.fNestedPaging
10617 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10618 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10619
10620 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10621 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10622 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10623
10624 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10625 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10626 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10627 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10628 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10629 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10630 break;
10631 }
10632
10633 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10634 {
10635 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10636 AssertRCReturn(rc, rc);
10637 rc = EMInterpretCLTS(pVM, pVCpu);
10638 AssertRCReturn(rc, rc);
10639 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10640 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10641 Log4(("CRX CLTS write rc=%d\n", rc));
10642 break;
10643 }
10644
10645 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10646 {
10647 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10648 AssertRCReturn(rc, rc);
10649 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10650 if (RT_LIKELY(rc == VINF_SUCCESS))
10651 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10653 Log4(("CRX LMSW write rc=%d\n", rc));
10654 break;
10655 }
10656
10657 default:
10658 {
10659 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10660 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10661 }
10662 }
10663
10664 /* Validate possible error codes. */
10665 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10666 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10667 if (RT_SUCCESS(rc))
10668 {
10669 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10670 AssertRCReturn(rc2, rc2);
10671 }
10672
10673 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10674 return rc;
10675}
10676
10677
10678/**
10679 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10680 * VM-exit.
10681 */
10682HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10683{
10684 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10685 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10686
10687 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10688 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10689 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10690 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10691 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10692 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10693 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10694 AssertRCReturn(rc2, rc2);
10695
10696 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10697 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10698 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10699 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10700 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10701 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10702 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10703 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10704
10705 /* I/O operation lookup arrays. */
10706 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10707 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10708
10709 VBOXSTRICTRC rcStrict;
10710 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10711 const uint32_t cbInstr = pVmxTransient->cbInstr;
10712 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10713 PVM pVM = pVCpu->CTX_SUFF(pVM);
10714 if (fIOString)
10715 {
10716#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10717 /*
10718 * INS/OUTS - I/O String instruction.
10719 *
10720 * Use instruction-information if available, otherwise fall back on
10721 * interpreting the instruction.
10722 */
10723 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10724 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10725 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10726 {
10727 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10728 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10729 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10730 AssertRCReturn(rc2, rc2);
10731 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10732 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10733 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10734 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10735 if (fIOWrite)
10736 {
10737 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10738 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10739 }
10740 else
10741 {
10742 /*
10743 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10744 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10745 * See Intel Instruction spec. for "INS".
10746 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10747 */
10748 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10749 }
10750 }
10751 else
10752 {
10753 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10754 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10755 AssertRCReturn(rc2, rc2);
10756 rcStrict = IEMExecOne(pVCpu);
10757 }
10758 /** @todo IEM needs to be setting these flags somehow. */
10759 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10760 fUpdateRipAlready = true;
10761#else
10762 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10763 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10764 if (RT_SUCCESS(rcStrict))
10765 {
10766 if (fIOWrite)
10767 {
10768 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10769 (DISCPUMODE)pDis->uAddrMode, cbValue);
10770 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10771 }
10772 else
10773 {
10774 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10775 (DISCPUMODE)pDis->uAddrMode, cbValue);
10776 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10777 }
10778 }
10779 else
10780 {
10781 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10782 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10783 }
10784#endif
10785 }
10786 else
10787 {
10788 /*
10789 * IN/OUT - I/O instruction.
10790 */
10791 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10792 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10793 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10794 if (fIOWrite)
10795 {
10796 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10797 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10798 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10800 }
10801 else
10802 {
10803 uint32_t u32Result = 0;
10804 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10805 if (IOM_SUCCESS(rcStrict))
10806 {
10807 /* Save result of I/O IN instr. in AL/AX/EAX. */
10808 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10809 }
10810 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10811 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10813 }
10814 }
10815
10816 if (IOM_SUCCESS(rcStrict))
10817 {
10818 if (!fUpdateRipAlready)
10819 {
10820 pMixedCtx->rip += cbInstr;
10821 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10822 }
10823
10824 /*
10825 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10826 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10827 */
10828 if (fIOString)
10829 {
10830 /** @todo Single-step for INS/OUTS with REP prefix? */
10831 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10832 }
10833 else if (fStepping)
10834 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10835
10836 /*
10837 * If any I/O breakpoints are armed, we need to check if one triggered
10838 * and take appropriate action.
10839 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10840 */
10841 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10842 AssertRCReturn(rc2, rc2);
10843
10844 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10845 * execution engines about whether hyper BPs and such are pending. */
10846 uint32_t const uDr7 = pMixedCtx->dr[7];
10847 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10848 && X86_DR7_ANY_RW_IO(uDr7)
10849 && (pMixedCtx->cr4 & X86_CR4_DE))
10850 || DBGFBpIsHwIoArmed(pVM)))
10851 {
10852 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10853
10854 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10855 VMMRZCallRing3Disable(pVCpu);
10856 HM_DISABLE_PREEMPT_IF_NEEDED();
10857
10858 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10859
10860 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10861 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10862 {
10863 /* Raise #DB. */
10864 if (fIsGuestDbgActive)
10865 ASMSetDR6(pMixedCtx->dr[6]);
10866 if (pMixedCtx->dr[7] != uDr7)
10867 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10868
10869 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10870 }
10871 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10872 else if ( rcStrict2 != VINF_SUCCESS
10873 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10874 rcStrict = rcStrict2;
10875
10876 HM_RESTORE_PREEMPT_IF_NEEDED();
10877 VMMRZCallRing3Enable(pVCpu);
10878 }
10879 }
10880
10881#ifdef DEBUG
10882 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10883 Assert(!fIOWrite);
10884 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10885 Assert(fIOWrite);
10886 else
10887 {
10888 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10889 * statuses, that the VMM device and some others may return. See
10890 * IOM_SUCCESS() for guidance. */
10891 AssertMsg( RT_FAILURE(rcStrict)
10892 || rcStrict == VINF_SUCCESS
10893 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10894 || rcStrict == VINF_EM_DBG_BREAKPOINT
10895 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10896 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10897 }
10898#endif
10899
10900 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10901 return VBOXSTRICTRC_TODO(rcStrict);
10902}
10903
10904
10905/**
10906 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10907 * VM-exit.
10908 */
10909HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10910{
10911 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10912
10913 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10914 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10915 AssertRCReturn(rc, rc);
10916 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10917 {
10918 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10919 AssertRCReturn(rc, rc);
10920 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10921 {
10922 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10923
10924 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10925 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10926
10927 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10928 Assert(!pVCpu->hm.s.Event.fPending);
10929 pVCpu->hm.s.Event.fPending = true;
10930 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10931 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10932 AssertRCReturn(rc, rc);
10933 if (fErrorCodeValid)
10934 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10935 else
10936 pVCpu->hm.s.Event.u32ErrCode = 0;
10937 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10938 && uVector == X86_XCPT_PF)
10939 {
10940 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10941 }
10942
10943 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10945 return VINF_EM_RAW_INJECT_TRPM_EVENT;
10946 }
10947 }
10948
10949 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10950 * emulation. */
10951 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10952 return VERR_EM_INTERPRETER;
10953}
10954
10955
10956/**
10957 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10958 */
10959HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10960{
10961 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10962 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10963 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10964 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10965 AssertRCReturn(rc, rc);
10966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10967 return VINF_EM_DBG_STEPPED;
10968}
10969
10970
10971/**
10972 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10973 */
10974HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10975{
10976 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10977
10978 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10979 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10980 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10981 return VINF_SUCCESS;
10982 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10983 return rc;
10984
10985#if 0
10986 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10987 * just sync the whole thing. */
10988 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10989#else
10990 /* Aggressive state sync. for now. */
10991 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10992 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10993 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10994#endif
10995 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10996 AssertRCReturn(rc, rc);
10997
10998 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10999 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11000 switch (uAccessType)
11001 {
11002 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11003 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11004 {
11005 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11006 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11007 {
11008 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11009 }
11010
11011 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11012 GCPhys &= PAGE_BASE_GC_MASK;
11013 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11014 PVM pVM = pVCpu->CTX_SUFF(pVM);
11015 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11016 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11017
11018 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11019 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11020 CPUMCTX2CORE(pMixedCtx), GCPhys);
11021 rc = VBOXSTRICTRC_VAL(rc2);
11022 Log4(("ApicAccess rc=%d\n", rc));
11023 if ( rc == VINF_SUCCESS
11024 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11025 || rc == VERR_PAGE_NOT_PRESENT)
11026 {
11027 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11028 | HM_CHANGED_GUEST_RSP
11029 | HM_CHANGED_GUEST_RFLAGS
11030 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11031 rc = VINF_SUCCESS;
11032 }
11033 break;
11034 }
11035
11036 default:
11037 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11038 rc = VINF_EM_RAW_EMULATE_INSTR;
11039 break;
11040 }
11041
11042 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11043 return rc;
11044}
11045
11046
11047/**
11048 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11049 * VM-exit.
11050 */
11051HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11052{
11053 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11054
11055 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11056 if (pVmxTransient->fWasGuestDebugStateActive)
11057 {
11058 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11059 HMVMX_RETURN_UNEXPECTED_EXIT();
11060 }
11061
11062 int rc = VERR_INTERNAL_ERROR_5;
11063 if ( !DBGFIsStepping(pVCpu)
11064 && !pVCpu->hm.s.fSingleInstruction
11065 && !pVmxTransient->fWasHyperDebugStateActive)
11066 {
11067 /* Don't intercept MOV DRx and #DB any more. */
11068 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11069 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11070 AssertRCReturn(rc, rc);
11071
11072 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11073 {
11074#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11075 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11076 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11077 AssertRCReturn(rc, rc);
11078#endif
11079 }
11080
11081 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11082 VMMRZCallRing3Disable(pVCpu);
11083 HM_DISABLE_PREEMPT_IF_NEEDED();
11084
11085 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11086 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11087 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11088
11089 HM_RESTORE_PREEMPT_IF_NEEDED();
11090 VMMRZCallRing3Enable(pVCpu);
11091
11092#ifdef VBOX_WITH_STATISTICS
11093 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11094 AssertRCReturn(rc, rc);
11095 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11097 else
11098 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11099#endif
11100 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11101 return VINF_SUCCESS;
11102 }
11103
11104 /*
11105 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11106 * Update the segment registers and DR7 from the CPU.
11107 */
11108 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11109 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11110 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11111 AssertRCReturn(rc, rc);
11112 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11113
11114 PVM pVM = pVCpu->CTX_SUFF(pVM);
11115 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11116 {
11117 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11118 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11119 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11120 if (RT_SUCCESS(rc))
11121 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11123 }
11124 else
11125 {
11126 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11127 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11128 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11130 }
11131
11132 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11133 if (RT_SUCCESS(rc))
11134 {
11135 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11136 AssertRCReturn(rc2, rc2);
11137 }
11138 return rc;
11139}
11140
11141
11142/**
11143 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11144 * Conditional VM-exit.
11145 */
11146HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11147{
11148 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11149 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11150
11151 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11152 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11153 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11154 return VINF_SUCCESS;
11155 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11156 return rc;
11157
11158 RTGCPHYS GCPhys = 0;
11159 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11160
11161#if 0
11162 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11163#else
11164 /* Aggressive state sync. for now. */
11165 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11166 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11167 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11168#endif
11169 AssertRCReturn(rc, rc);
11170
11171 /*
11172 * If we succeed, resume guest execution.
11173 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11174 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11175 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11176 * weird case. See @bugref{6043}.
11177 */
11178 PVM pVM = pVCpu->CTX_SUFF(pVM);
11179 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11180 rc = VBOXSTRICTRC_VAL(rc2);
11181 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11182 if ( rc == VINF_SUCCESS
11183 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11184 || rc == VERR_PAGE_NOT_PRESENT)
11185 {
11186 /* Successfully handled MMIO operation. */
11187 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11188 | HM_CHANGED_GUEST_RSP
11189 | HM_CHANGED_GUEST_RFLAGS
11190 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11191 rc = VINF_SUCCESS;
11192 }
11193 return rc;
11194}
11195
11196
11197/**
11198 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11199 * VM-exit.
11200 */
11201HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11202{
11203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11204 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11205
11206 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11207 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11208 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11209 return VINF_SUCCESS;
11210 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11211 return rc;
11212
11213 RTGCPHYS GCPhys = 0;
11214 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11215 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11216#if 0
11217 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11218#else
11219 /* Aggressive state sync. for now. */
11220 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11221 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11222 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11223#endif
11224 AssertRCReturn(rc, rc);
11225
11226 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11227 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11228
11229 RTGCUINT uErrorCode = 0;
11230 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11231 uErrorCode |= X86_TRAP_PF_ID;
11232 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11233 uErrorCode |= X86_TRAP_PF_RW;
11234 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11235 uErrorCode |= X86_TRAP_PF_P;
11236
11237 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11238
11239 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11240 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11241
11242 /* Handle the pagefault trap for the nested shadow table. */
11243 PVM pVM = pVCpu->CTX_SUFF(pVM);
11244 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11245 TRPMResetTrap(pVCpu);
11246
11247 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11248 if ( rc == VINF_SUCCESS
11249 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11250 || rc == VERR_PAGE_NOT_PRESENT)
11251 {
11252 /* Successfully synced our nested page tables. */
11253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11254 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11255 | HM_CHANGED_GUEST_RSP
11256 | HM_CHANGED_GUEST_RFLAGS);
11257 return VINF_SUCCESS;
11258 }
11259
11260 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11261 return rc;
11262}
11263
11264/** @} */
11265
11266/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11267/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11268/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11269
11270/** @name VM-exit exception handlers.
11271 * @{
11272 */
11273
11274/**
11275 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11276 */
11277static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11278{
11279 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11281
11282 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11283 AssertRCReturn(rc, rc);
11284
11285 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11286 {
11287 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11288 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11289
11290 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11291 * provides VM-exit instruction length. If this causes problem later,
11292 * disassemble the instruction like it's done on AMD-V. */
11293 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11294 AssertRCReturn(rc2, rc2);
11295 return rc;
11296 }
11297
11298 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11299 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11300 return rc;
11301}
11302
11303
11304/**
11305 * VM-exit exception handler for #BP (Breakpoint exception).
11306 */
11307static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11308{
11309 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11311
11312 /** @todo Try optimize this by not saving the entire guest state unless
11313 * really needed. */
11314 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11315 AssertRCReturn(rc, rc);
11316
11317 PVM pVM = pVCpu->CTX_SUFF(pVM);
11318 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11319 if (rc == VINF_EM_RAW_GUEST_TRAP)
11320 {
11321 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11322 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11323 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11324 AssertRCReturn(rc, rc);
11325
11326 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11327 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11328 }
11329
11330 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11331 return rc;
11332}
11333
11334
11335/**
11336 * VM-exit exception handler for #DB (Debug exception).
11337 */
11338static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11339{
11340 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11341 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11342 Log6(("XcptDB\n"));
11343
11344 /*
11345 * Get the DR6-like values from the exit qualification and pass it to DBGF
11346 * for processing.
11347 */
11348 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11349 AssertRCReturn(rc, rc);
11350
11351 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11352 uint64_t uDR6 = X86_DR6_INIT_VAL;
11353 uDR6 |= ( pVmxTransient->uExitQualification
11354 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11355
11356 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11357 if (rc == VINF_EM_RAW_GUEST_TRAP)
11358 {
11359 /*
11360 * The exception was for the guest. Update DR6, DR7.GD and
11361 * IA32_DEBUGCTL.LBR before forwarding it.
11362 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11363 */
11364 VMMRZCallRing3Disable(pVCpu);
11365 HM_DISABLE_PREEMPT_IF_NEEDED();
11366
11367 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11368 pMixedCtx->dr[6] |= uDR6;
11369 if (CPUMIsGuestDebugStateActive(pVCpu))
11370 ASMSetDR6(pMixedCtx->dr[6]);
11371
11372 HM_RESTORE_PREEMPT_IF_NEEDED();
11373 VMMRZCallRing3Enable(pVCpu);
11374
11375 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11376 AssertRCReturn(rc, rc);
11377
11378 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11379 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11380
11381 /* Paranoia. */
11382 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11383 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11384
11385 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11386 AssertRCReturn(rc, rc);
11387
11388 /*
11389 * Raise #DB in the guest.
11390 *
11391 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11392 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11393 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11394 *
11395 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11396 */
11397 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11398 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11399 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11400 AssertRCReturn(rc, rc);
11401 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11402 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11403 return VINF_SUCCESS;
11404 }
11405
11406 /*
11407 * Not a guest trap, must be a hypervisor related debug event then.
11408 * Update DR6 in case someone is interested in it.
11409 */
11410 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11411 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11412 CPUMSetHyperDR6(pVCpu, uDR6);
11413
11414 return rc;
11415}
11416
11417
11418/**
11419 * VM-exit exception handler for #NM (Device-not-available exception: floating
11420 * point exception).
11421 */
11422static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11423{
11424 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11425
11426 /* We require CR0 and EFER. EFER is always up-to-date. */
11427 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11428 AssertRCReturn(rc, rc);
11429
11430 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11431 VMMRZCallRing3Disable(pVCpu);
11432 HM_DISABLE_PREEMPT_IF_NEEDED();
11433
11434 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11435 if (pVmxTransient->fWasGuestFPUStateActive)
11436 {
11437 rc = VINF_EM_RAW_GUEST_TRAP;
11438 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11439 }
11440 else
11441 {
11442#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11443 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11444#endif
11445 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11446 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11447 }
11448
11449 HM_RESTORE_PREEMPT_IF_NEEDED();
11450 VMMRZCallRing3Enable(pVCpu);
11451
11452 if (rc == VINF_SUCCESS)
11453 {
11454 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11455 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11456 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11457 pVCpu->hm.s.fUseGuestFpu = true;
11458 }
11459 else
11460 {
11461 /* Forward #NM to the guest. */
11462 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11463 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11464 AssertRCReturn(rc, rc);
11465 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11466 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11467 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11468 }
11469
11470 return VINF_SUCCESS;
11471}
11472
11473
11474/**
11475 * VM-exit exception handler for #GP (General-protection exception).
11476 *
11477 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11478 */
11479static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11480{
11481 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11482 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11483
11484 int rc = VERR_INTERNAL_ERROR_5;
11485 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11486 {
11487#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11488 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11489 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11490 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11491 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11492 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11493 AssertRCReturn(rc, rc);
11494 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11495 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11496 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11497 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11498 return rc;
11499#else
11500 /* We don't intercept #GP. */
11501 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11502 NOREF(pVmxTransient);
11503 return VERR_VMX_UNEXPECTED_EXCEPTION;
11504#endif
11505 }
11506
11507 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11508 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11509
11510 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11511 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11512 AssertRCReturn(rc, rc);
11513
11514 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11515 uint32_t cbOp = 0;
11516 PVM pVM = pVCpu->CTX_SUFF(pVM);
11517 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11518 if (RT_SUCCESS(rc))
11519 {
11520 rc = VINF_SUCCESS;
11521 Assert(cbOp == pDis->cbInstr);
11522 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11523 switch (pDis->pCurInstr->uOpcode)
11524 {
11525 case OP_CLI:
11526 {
11527 pMixedCtx->eflags.Bits.u1IF = 0;
11528 pMixedCtx->rip += pDis->cbInstr;
11529 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11530 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11531 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11532 break;
11533 }
11534
11535 case OP_STI:
11536 {
11537 pMixedCtx->eflags.Bits.u1IF = 1;
11538 pMixedCtx->rip += pDis->cbInstr;
11539 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11540 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11541 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11542 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11543 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11544 break;
11545 }
11546
11547 case OP_HLT:
11548 {
11549 rc = VINF_EM_HALT;
11550 pMixedCtx->rip += pDis->cbInstr;
11551 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11552 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11553 break;
11554 }
11555
11556 case OP_POPF:
11557 {
11558 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11559 uint32_t cbParm;
11560 uint32_t uMask;
11561 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11562 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11563 {
11564 cbParm = 4;
11565 uMask = 0xffffffff;
11566 }
11567 else
11568 {
11569 cbParm = 2;
11570 uMask = 0xffff;
11571 }
11572
11573 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11574 RTGCPTR GCPtrStack = 0;
11575 X86EFLAGS Eflags;
11576 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11577 &GCPtrStack);
11578 if (RT_SUCCESS(rc))
11579 {
11580 Assert(sizeof(Eflags.u32) >= cbParm);
11581 Eflags.u32 = 0;
11582 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11583 }
11584 if (RT_FAILURE(rc))
11585 {
11586 rc = VERR_EM_INTERPRETER;
11587 break;
11588 }
11589 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11590 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11591 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11592 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11593 pMixedCtx->esp += cbParm;
11594 pMixedCtx->esp &= uMask;
11595 pMixedCtx->rip += pDis->cbInstr;
11596 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11597 | HM_CHANGED_GUEST_RSP
11598 | HM_CHANGED_GUEST_RFLAGS);
11599 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11600 if (fStepping)
11601 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11602
11603 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11604 break;
11605 }
11606
11607 case OP_PUSHF:
11608 {
11609 uint32_t cbParm;
11610 uint32_t uMask;
11611 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11612 {
11613 cbParm = 4;
11614 uMask = 0xffffffff;
11615 }
11616 else
11617 {
11618 cbParm = 2;
11619 uMask = 0xffff;
11620 }
11621
11622 /* Get the stack pointer & push the contents of eflags onto the stack. */
11623 RTGCPTR GCPtrStack = 0;
11624 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11625 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11626 if (RT_FAILURE(rc))
11627 {
11628 rc = VERR_EM_INTERPRETER;
11629 break;
11630 }
11631 X86EFLAGS Eflags = pMixedCtx->eflags;
11632 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11633 Eflags.Bits.u1RF = 0;
11634 Eflags.Bits.u1VM = 0;
11635
11636 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11637 if (RT_FAILURE(rc))
11638 {
11639 rc = VERR_EM_INTERPRETER;
11640 break;
11641 }
11642 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11643 pMixedCtx->esp -= cbParm;
11644 pMixedCtx->esp &= uMask;
11645 pMixedCtx->rip += pDis->cbInstr;
11646 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11647 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11648 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11649 break;
11650 }
11651
11652 case OP_IRET:
11653 {
11654 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11655 * instruction reference. */
11656 RTGCPTR GCPtrStack = 0;
11657 uint32_t uMask = 0xffff;
11658 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11659 uint16_t aIretFrame[3];
11660 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11661 {
11662 rc = VERR_EM_INTERPRETER;
11663 break;
11664 }
11665 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11666 &GCPtrStack);
11667 if (RT_SUCCESS(rc))
11668 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11669 if (RT_FAILURE(rc))
11670 {
11671 rc = VERR_EM_INTERPRETER;
11672 break;
11673 }
11674 pMixedCtx->eip = 0;
11675 pMixedCtx->ip = aIretFrame[0];
11676 pMixedCtx->cs.Sel = aIretFrame[1];
11677 pMixedCtx->cs.ValidSel = aIretFrame[1];
11678 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11679 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11680 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11681 pMixedCtx->sp += sizeof(aIretFrame);
11682 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11683 | HM_CHANGED_GUEST_SEGMENT_REGS
11684 | HM_CHANGED_GUEST_RSP
11685 | HM_CHANGED_GUEST_RFLAGS);
11686 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11687 if (fStepping)
11688 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11689 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11690 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11691 break;
11692 }
11693
11694 case OP_INT:
11695 {
11696 uint16_t uVector = pDis->Param1.uValue & 0xff;
11697 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11698 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11699 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11700 break;
11701 }
11702
11703 case OP_INTO:
11704 {
11705 if (pMixedCtx->eflags.Bits.u1OF)
11706 {
11707 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11708 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11709 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11710 }
11711 break;
11712 }
11713
11714 default:
11715 {
11716 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11717 EMCODETYPE_SUPERVISOR);
11718 rc = VBOXSTRICTRC_VAL(rc2);
11719 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11720 /** @todo We have to set pending-debug exceptions here when the guest is
11721 * single-stepping depending on the instruction that was interpreted. */
11722 Log4(("#GP rc=%Rrc\n", rc));
11723 break;
11724 }
11725 }
11726 }
11727 else
11728 rc = VERR_EM_INTERPRETER;
11729
11730 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11731 ("#GP Unexpected rc=%Rrc\n", rc));
11732 return rc;
11733}
11734
11735
11736#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11737/**
11738 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11739 * the exception reported in the VMX transient structure back into the VM.
11740 *
11741 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11742 * up-to-date.
11743 */
11744static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11745{
11746 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11747
11748 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11749 hmR0VmxCheckExitDueToEventDelivery(). */
11750 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11751 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11752 AssertRCReturn(rc, rc);
11753 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11754
11755#ifdef DEBUG_ramshankar
11756 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11757 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11758 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11759#endif
11760
11761 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11762 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11763 return VINF_SUCCESS;
11764}
11765#endif
11766
11767
11768/**
11769 * VM-exit exception handler for #PF (Page-fault exception).
11770 */
11771static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11772{
11773 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11774 PVM pVM = pVCpu->CTX_SUFF(pVM);
11775 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11776 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11777 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11778 AssertRCReturn(rc, rc);
11779
11780#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11781 if (pVM->hm.s.fNestedPaging)
11782 {
11783 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11784 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11785 {
11786 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11787 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11788 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11789 }
11790 else
11791 {
11792 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11793 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11794 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11795 }
11796 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11797 return rc;
11798 }
11799#else
11800 Assert(!pVM->hm.s.fNestedPaging);
11801 NOREF(pVM);
11802#endif
11803
11804 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11805 AssertRCReturn(rc, rc);
11806
11807 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11808 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11809
11810 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11811 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11812 (RTGCPTR)pVmxTransient->uExitQualification);
11813
11814 Log4(("#PF: rc=%Rrc\n", rc));
11815 if (rc == VINF_SUCCESS)
11816 {
11817 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11818 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11819 * memory? We don't update the whole state here... */
11820 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11821 | HM_CHANGED_GUEST_RSP
11822 | HM_CHANGED_GUEST_RFLAGS
11823 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11824 TRPMResetTrap(pVCpu);
11825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11826 return rc;
11827 }
11828 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11829 {
11830 if (!pVmxTransient->fVectoringPF)
11831 {
11832 /* It's a guest page fault and needs to be reflected to the guest. */
11833 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11834 TRPMResetTrap(pVCpu);
11835 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11836 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11837 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11838 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11839 }
11840 else
11841 {
11842 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11843 TRPMResetTrap(pVCpu);
11844 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11845 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11846 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11847 }
11848
11849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11850 return VINF_SUCCESS;
11851 }
11852
11853 TRPMResetTrap(pVCpu);
11854 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11855 return rc;
11856}
11857
11858/** @} */
11859
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette