VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 52655

Last change on this file since 52655 was 52655, checked in by vboxsync, 10 years ago

VMM/HMVMXR0,HMSVMR0: oops, forgot bugref number.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 499.2 KB
Line 
1/* $Id: HMVMXR0.cpp 52655 2014-09-09 14:03:59Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringDoublePF;
282 /** Whether the VM-exit was caused by a page-fault during delivery of an
283 * external interrupt or NMI. */
284 bool fVectoringPF;
285} VMXTRANSIENT;
286AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
287AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
288AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
289AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
290AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291/** Pointer to VMX transient state. */
292typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294
295/**
296 * MSR-bitmap read permissions.
297 */
298typedef enum VMXMSREXITREAD
299{
300 /** Reading this MSR causes a VM-exit. */
301 VMXMSREXIT_INTERCEPT_READ = 0xb,
302 /** Reading this MSR does not cause a VM-exit. */
303 VMXMSREXIT_PASSTHRU_READ
304} VMXMSREXITREAD;
305/** Pointer to MSR-bitmap read permissions. */
306typedef VMXMSREXITREAD* PVMXMSREXITREAD;
307
308/**
309 * MSR-bitmap write permissions.
310 */
311typedef enum VMXMSREXITWRITE
312{
313 /** Writing to this MSR causes a VM-exit. */
314 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
315 /** Writing to this MSR does not cause a VM-exit. */
316 VMXMSREXIT_PASSTHRU_WRITE
317} VMXMSREXITWRITE;
318/** Pointer to MSR-bitmap write permissions. */
319typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
320
321
322/**
323 * VMX VM-exit handler.
324 *
325 * @returns VBox status code.
326 * @param pVCpu Pointer to the VMCPU.
327 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
328 * out-of-sync. Make sure to update the required
329 * fields before using them.
330 * @param pVmxTransient Pointer to the VMX-transient structure.
331 */
332#ifndef HMVMX_USE_FUNCTION_TABLE
333typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
334#else
335typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
336/** Pointer to VM-exit handler. */
337typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
338#endif
339
340
341/*******************************************************************************
342* Internal Functions *
343*******************************************************************************/
344static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
345static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
346static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
347 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
348#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
349static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
350#endif
351#ifndef HMVMX_USE_FUNCTION_TABLE
352DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
353# define HMVMX_EXIT_DECL static int
354#else
355# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
356#endif
357
358/** @name VM-exit handlers.
359 * @{
360 */
361static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
365static FNVMXEXITHANDLER hmR0VmxExitSipi;
366static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
367static FNVMXEXITHANDLER hmR0VmxExitSmi;
368static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
369static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
370static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373static FNVMXEXITHANDLER hmR0VmxExitHlt;
374static FNVMXEXITHANDLER hmR0VmxExitInvd;
375static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
379static FNVMXEXITHANDLER hmR0VmxExitRsm;
380static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
381static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
382static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
383static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
384static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
385static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
386static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
387static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
388static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
389static FNVMXEXITHANDLER hmR0VmxExitMwait;
390static FNVMXEXITHANDLER hmR0VmxExitMtf;
391static FNVMXEXITHANDLER hmR0VmxExitMonitor;
392static FNVMXEXITHANDLER hmR0VmxExitPause;
393static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
394static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
395static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
396static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
397static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
398static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
399static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
400static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
401static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
402static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
403static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
404static FNVMXEXITHANDLER hmR0VmxExitRdrand;
405static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
406/** @} */
407
408static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
415static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416#endif
417static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
418
419/*******************************************************************************
420* Global Variables *
421*******************************************************************************/
422#ifdef HMVMX_USE_FUNCTION_TABLE
423
424/**
425 * VMX_EXIT dispatch table.
426 */
427static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
428{
429 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
430 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
431 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
432 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
433 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
434 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
435 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
436 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
437 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
438 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
439 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
440 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
441 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
442 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
443 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
444 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
445 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
446 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
447 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
448 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
449 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
450 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
451 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
452 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
453 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
454 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
455 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
456 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
457 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
458 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
459 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
460 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
461 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
462 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
463 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
464 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
465 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
466 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
467 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
468 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
469 /* 40 UNDEFINED */ hmR0VmxExitPause,
470 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
471 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
472 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
473 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
474 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
475 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
476 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
477 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
478 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
479 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
480 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
481 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
482 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
483 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
484 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
485 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
486 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
487 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
488 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
489};
490#endif /* HMVMX_USE_FUNCTION_TABLE */
491
492#ifdef VBOX_STRICT
493static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
494{
495 /* 0 */ "(Not Used)",
496 /* 1 */ "VMCALL executed in VMX root operation.",
497 /* 2 */ "VMCLEAR with invalid physical address.",
498 /* 3 */ "VMCLEAR with VMXON pointer.",
499 /* 4 */ "VMLAUNCH with non-clear VMCS.",
500 /* 5 */ "VMRESUME with non-launched VMCS.",
501 /* 6 */ "VMRESUME after VMXOFF",
502 /* 7 */ "VM entry with invalid control fields.",
503 /* 8 */ "VM entry with invalid host state fields.",
504 /* 9 */ "VMPTRLD with invalid physical address.",
505 /* 10 */ "VMPTRLD with VMXON pointer.",
506 /* 11 */ "VMPTRLD with incorrect revision identifier.",
507 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
508 /* 13 */ "VMWRITE to read-only VMCS component.",
509 /* 14 */ "(Not Used)",
510 /* 15 */ "VMXON executed in VMX root operation.",
511 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
512 /* 17 */ "VM entry with non-launched executing VMCS.",
513 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
514 /* 19 */ "VMCALL with non-clear VMCS.",
515 /* 20 */ "VMCALL with invalid VM-exit control fields.",
516 /* 21 */ "(Not Used)",
517 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
518 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
519 /* 24 */ "VMCALL with invalid SMM-monitor features.",
520 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
521 /* 26 */ "VM entry with events blocked by MOV SS.",
522 /* 27 */ "(Not Used)",
523 /* 28 */ "Invalid operand to INVEPT/INVVPID."
524};
525#endif /* VBOX_STRICT */
526
527
528
529/**
530 * Updates the VM's last error record. If there was a VMX instruction error,
531 * reads the error data from the VMCS and updates VCPU's last error record as
532 * well.
533 *
534 * @param pVM Pointer to the VM.
535 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
536 * VERR_VMX_UNABLE_TO_START_VM or
537 * VERR_VMX_INVALID_VMCS_FIELD).
538 * @param rc The error code.
539 */
540static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
541{
542 AssertPtr(pVM);
543 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
544 || rc == VERR_VMX_UNABLE_TO_START_VM)
545 {
546 AssertPtrReturnVoid(pVCpu);
547 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
548 }
549 pVM->hm.s.lLastError = rc;
550}
551
552
553/**
554 * Reads the VM-entry interruption-information field from the VMCS into the VMX
555 * transient structure.
556 *
557 * @returns VBox status code.
558 * @param pVmxTransient Pointer to the VMX transient structure.
559 *
560 * @remarks No-long-jump zone!!!
561 */
562DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
563{
564 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
565 AssertRCReturn(rc, rc);
566 return VINF_SUCCESS;
567}
568
569
570/**
571 * Reads the VM-entry exception error code field from the VMCS into
572 * the VMX transient structure.
573 *
574 * @returns VBox status code.
575 * @param pVmxTransient Pointer to the VMX transient structure.
576 *
577 * @remarks No-long-jump zone!!!
578 */
579DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
580{
581 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
582 AssertRCReturn(rc, rc);
583 return VINF_SUCCESS;
584}
585
586
587/**
588 * Reads the VM-entry exception error code field from the VMCS into
589 * the VMX transient structure.
590 *
591 * @returns VBox status code.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-exit interruption-information field from the VMCS into the VMX
606 * transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVmxTransient Pointer to the VMX transient structure.
610 */
611DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
612{
613 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
614 {
615 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
616 AssertRCReturn(rc, rc);
617 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
618 }
619 return VINF_SUCCESS;
620}
621
622
623/**
624 * Reads the VM-exit interruption error code from the VMCS into the VMX
625 * transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 */
630DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
631{
632 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
633 {
634 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
635 AssertRCReturn(rc, rc);
636 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
637 }
638 return VINF_SUCCESS;
639}
640
641
642/**
643 * Reads the VM-exit instruction length field from the VMCS into the VMX
644 * transient structure.
645 *
646 * @returns VBox status code.
647 * @param pVCpu Pointer to the VMCPU.
648 * @param pVmxTransient Pointer to the VMX transient structure.
649 */
650DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
651{
652 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
653 {
654 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
655 AssertRCReturn(rc, rc);
656 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
657 }
658 return VINF_SUCCESS;
659}
660
661
662/**
663 * Reads the VM-exit instruction-information field from the VMCS into
664 * the VMX transient structure.
665 *
666 * @returns VBox status code.
667 * @param pVmxTransient Pointer to the VMX transient structure.
668 */
669DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
670{
671 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
672 {
673 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
674 AssertRCReturn(rc, rc);
675 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
676 }
677 return VINF_SUCCESS;
678}
679
680
681/**
682 * Reads the exit qualification from the VMCS into the VMX transient structure.
683 *
684 * @returns VBox status code.
685 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
686 * case).
687 * @param pVmxTransient Pointer to the VMX transient structure.
688 */
689DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
690{
691 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
692 {
693 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
694 AssertRCReturn(rc, rc);
695 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
696 }
697 return VINF_SUCCESS;
698}
699
700
701/**
702 * Reads the IDT-vectoring information field from the VMCS into the VMX
703 * transient structure.
704 *
705 * @returns VBox status code.
706 * @param pVmxTransient Pointer to the VMX transient structure.
707 *
708 * @remarks No-long-jump zone!!!
709 */
710DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
711{
712 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
713 {
714 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
715 AssertRCReturn(rc, rc);
716 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
717 }
718 return VINF_SUCCESS;
719}
720
721
722/**
723 * Reads the IDT-vectoring error code from the VMCS into the VMX
724 * transient structure.
725 *
726 * @returns VBox status code.
727 * @param pVmxTransient Pointer to the VMX transient structure.
728 */
729DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
730{
731 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
732 {
733 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
734 AssertRCReturn(rc, rc);
735 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
736 }
737 return VINF_SUCCESS;
738}
739
740
741/**
742 * Enters VMX root mode operation on the current CPU.
743 *
744 * @returns VBox status code.
745 * @param pVM Pointer to the VM (optional, can be NULL, after
746 * a resume).
747 * @param HCPhysCpuPage Physical address of the VMXON region.
748 * @param pvCpuPage Pointer to the VMXON region.
749 */
750static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
751{
752 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
753 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
754 Assert(pvCpuPage);
755 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
756
757 if (pVM)
758 {
759 /* Write the VMCS revision dword to the VMXON region. */
760 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
761 }
762
763 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
764 RTCCUINTREG uEflags = ASMIntDisableFlags();
765
766 /* Enable the VMX bit in CR4 if necessary. */
767 RTCCUINTREG uCr4 = ASMGetCR4();
768 if (!(uCr4 & X86_CR4_VMXE))
769 ASMSetCR4(uCr4 | X86_CR4_VMXE);
770
771 /* Enter VMX root mode. */
772 int rc = VMXEnable(HCPhysCpuPage);
773 if (RT_FAILURE(rc))
774 ASMSetCR4(uCr4);
775
776 /* Restore interrupts. */
777 ASMSetFlags(uEflags);
778 return rc;
779}
780
781
782/**
783 * Exits VMX root mode operation on the current CPU.
784 *
785 * @returns VBox status code.
786 */
787static int hmR0VmxLeaveRootMode(void)
788{
789 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
790
791 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
792 RTCCUINTREG uEflags = ASMIntDisableFlags();
793
794 /* If we're for some reason not in VMX root mode, then don't leave it. */
795 RTCCUINTREG uHostCR4 = ASMGetCR4();
796
797 int rc;
798 if (uHostCR4 & X86_CR4_VMXE)
799 {
800 /* Exit VMX root mode and clear the VMX bit in CR4. */
801 VMXDisable();
802 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
803 rc = VINF_SUCCESS;
804 }
805 else
806 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
807
808 /* Restore interrupts. */
809 ASMSetFlags(uEflags);
810 return rc;
811}
812
813
814/**
815 * Allocates and maps one physically contiguous page. The allocated page is
816 * zero'd out. (Used by various VT-x structures).
817 *
818 * @returns IPRT status code.
819 * @param pMemObj Pointer to the ring-0 memory object.
820 * @param ppVirt Where to store the virtual address of the
821 * allocation.
822 * @param pPhys Where to store the physical address of the
823 * allocation.
824 */
825DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
826{
827 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
828 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
829 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
830
831 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
832 if (RT_FAILURE(rc))
833 return rc;
834 *ppVirt = RTR0MemObjAddress(*pMemObj);
835 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
836 ASMMemZero32(*ppVirt, PAGE_SIZE);
837 return VINF_SUCCESS;
838}
839
840
841/**
842 * Frees and unmaps an allocated physical page.
843 *
844 * @param pMemObj Pointer to the ring-0 memory object.
845 * @param ppVirt Where to re-initialize the virtual address of
846 * allocation as 0.
847 * @param pHCPhys Where to re-initialize the physical address of the
848 * allocation as 0.
849 */
850DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
851{
852 AssertPtr(pMemObj);
853 AssertPtr(ppVirt);
854 AssertPtr(pHCPhys);
855 if (*pMemObj != NIL_RTR0MEMOBJ)
856 {
857 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
858 AssertRC(rc);
859 *pMemObj = NIL_RTR0MEMOBJ;
860 *ppVirt = 0;
861 *pHCPhys = 0;
862 }
863}
864
865
866/**
867 * Worker function to free VT-x related structures.
868 *
869 * @returns IPRT status code.
870 * @param pVM Pointer to the VM.
871 */
872static void hmR0VmxStructsFree(PVM pVM)
873{
874 for (VMCPUID i = 0; i < pVM->cCpus; i++)
875 {
876 PVMCPU pVCpu = &pVM->aCpus[i];
877 AssertPtr(pVCpu);
878
879 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
881
882 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
884
885 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
887 }
888
889 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
890#ifdef VBOX_WITH_CRASHDUMP_MAGIC
891 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
892#endif
893}
894
895
896/**
897 * Worker function to allocate VT-x related VM structures.
898 *
899 * @returns IPRT status code.
900 * @param pVM Pointer to the VM.
901 */
902static int hmR0VmxStructsAlloc(PVM pVM)
903{
904 /*
905 * Initialize members up-front so we can cleanup properly on allocation failure.
906 */
907#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
908 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
909 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
910 pVM->hm.s.vmx.HCPhys##a_Name = 0;
911
912#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
913 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
914 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
915 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
916
917#ifdef VBOX_WITH_CRASHDUMP_MAGIC
918 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
919#endif
920 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
921
922 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
923 for (VMCPUID i = 0; i < pVM->cCpus; i++)
924 {
925 PVMCPU pVCpu = &pVM->aCpus[i];
926 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
927 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
928 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
929 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
931 }
932#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
933#undef VMXLOCAL_INIT_VM_MEMOBJ
934
935 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
936 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
937 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
938 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
939
940 /*
941 * Allocate all the VT-x structures.
942 */
943 int rc = VINF_SUCCESS;
944#ifdef VBOX_WITH_CRASHDUMP_MAGIC
945 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
946 if (RT_FAILURE(rc))
947 goto cleanup;
948 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
949 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
950#endif
951
952 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
953 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
954 {
955 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
956 &pVM->hm.s.vmx.HCPhysApicAccess);
957 if (RT_FAILURE(rc))
958 goto cleanup;
959 }
960
961 /*
962 * Initialize per-VCPU VT-x structures.
963 */
964 for (VMCPUID i = 0; i < pVM->cCpus; i++)
965 {
966 PVMCPU pVCpu = &pVM->aCpus[i];
967 AssertPtr(pVCpu);
968
969 /* Allocate the VM control structure (VMCS). */
970 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
971 if (RT_FAILURE(rc))
972 goto cleanup;
973
974 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
975 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
976 {
977 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
978 &pVCpu->hm.s.vmx.HCPhysVirtApic);
979 if (RT_FAILURE(rc))
980 goto cleanup;
981 }
982
983 /*
984 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
985 * transparent accesses of specific MSRs.
986 *
987 * If the condition for enabling MSR bitmaps changes here, don't forget to
988 * update HMIsMsrBitmapsAvailable().
989 */
990 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
991 {
992 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
993 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
994 if (RT_FAILURE(rc))
995 goto cleanup;
996 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
997 }
998
999 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003
1004 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1005 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008 }
1009
1010 return VINF_SUCCESS;
1011
1012cleanup:
1013 hmR0VmxStructsFree(pVM);
1014 return rc;
1015}
1016
1017
1018/**
1019 * Does global VT-x initialization (called during module initialization).
1020 *
1021 * @returns VBox status code.
1022 */
1023VMMR0DECL(int) VMXR0GlobalInit(void)
1024{
1025#ifdef HMVMX_USE_FUNCTION_TABLE
1026 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1027# ifdef VBOX_STRICT
1028 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1029 Assert(g_apfnVMExitHandlers[i]);
1030# endif
1031#endif
1032 return VINF_SUCCESS;
1033}
1034
1035
1036/**
1037 * Does global VT-x termination (called during module termination).
1038 */
1039VMMR0DECL(void) VMXR0GlobalTerm()
1040{
1041 /* Nothing to do currently. */
1042}
1043
1044
1045/**
1046 * Sets up and activates VT-x on the current CPU.
1047 *
1048 * @returns VBox status code.
1049 * @param pCpu Pointer to the global CPU info struct.
1050 * @param pVM Pointer to the VM (can be NULL after a host resume
1051 * operation).
1052 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1053 * fEnabledByHost is true).
1054 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1055 * @a fEnabledByHost is true).
1056 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1057 * enable VT-x on the host.
1058 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1059 */
1060VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1061 void *pvMsrs)
1062{
1063 Assert(pCpu);
1064 Assert(pvMsrs);
1065 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1066
1067 /* Enable VT-x if it's not already enabled by the host. */
1068 if (!fEnabledByHost)
1069 {
1070 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1071 if (RT_FAILURE(rc))
1072 return rc;
1073 }
1074
1075 /*
1076 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1077 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1078 */
1079 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1080 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1081 {
1082 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1083 pCpu->fFlushAsidBeforeUse = false;
1084 }
1085 else
1086 pCpu->fFlushAsidBeforeUse = true;
1087
1088 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1089 ++pCpu->cTlbFlushes;
1090
1091 return VINF_SUCCESS;
1092}
1093
1094
1095/**
1096 * Deactivates VT-x on the current CPU.
1097 *
1098 * @returns VBox status code.
1099 * @param pCpu Pointer to the global CPU info struct.
1100 * @param pvCpuPage Pointer to the VMXON region.
1101 * @param HCPhysCpuPage Physical address of the VMXON region.
1102 *
1103 * @remarks This function should never be called when SUPR0EnableVTx() or
1104 * similar was used to enable VT-x on the host.
1105 */
1106VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1107{
1108 NOREF(pCpu);
1109 NOREF(pvCpuPage);
1110 NOREF(HCPhysCpuPage);
1111
1112 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1113 return hmR0VmxLeaveRootMode();
1114}
1115
1116
1117/**
1118 * Sets the permission bits for the specified MSR in the MSR bitmap.
1119 *
1120 * @param pVCpu Pointer to the VMCPU.
1121 * @param uMSR The MSR value.
1122 * @param enmRead Whether reading this MSR causes a VM-exit.
1123 * @param enmWrite Whether writing this MSR causes a VM-exit.
1124 */
1125static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1126{
1127 int32_t iBit;
1128 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1129
1130 /*
1131 * Layout:
1132 * 0x000 - 0x3ff - Low MSR read bits
1133 * 0x400 - 0x7ff - High MSR read bits
1134 * 0x800 - 0xbff - Low MSR write bits
1135 * 0xc00 - 0xfff - High MSR write bits
1136 */
1137 if (uMsr <= 0x00001FFF)
1138 iBit = uMsr;
1139 else if ( uMsr >= 0xC0000000
1140 && uMsr <= 0xC0001FFF)
1141 {
1142 iBit = (uMsr - 0xC0000000);
1143 pbMsrBitmap += 0x400;
1144 }
1145 else
1146 {
1147 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1148 return;
1149 }
1150
1151 Assert(iBit <= 0x1fff);
1152 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1153 ASMBitSet(pbMsrBitmap, iBit);
1154 else
1155 ASMBitClear(pbMsrBitmap, iBit);
1156
1157 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1158 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1159 else
1160 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1161}
1162
1163
1164#ifdef VBOX_STRICT
1165/**
1166 * Gets the permission bits for the specified MSR in the MSR bitmap.
1167 *
1168 * @returns VBox status code.
1169 * @retval VINF_SUCCESS if the specified MSR is found.
1170 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1171 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1172 *
1173 * @param pVCpu Pointer to the VMCPU.
1174 * @param uMsr The MSR.
1175 * @param penmRead Where to store the read permissions.
1176 * @param penmWrite Where to store the write permissions.
1177 */
1178static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1179{
1180 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1181 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1182 int32_t iBit;
1183 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1184
1185 /* See hmR0VmxSetMsrPermission() for the layout. */
1186 if (uMsr <= 0x00001FFF)
1187 iBit = uMsr;
1188 else if ( uMsr >= 0xC0000000
1189 && uMsr <= 0xC0001FFF)
1190 {
1191 iBit = (uMsr - 0xC0000000);
1192 pbMsrBitmap += 0x400;
1193 }
1194 else
1195 {
1196 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1197 return VERR_NOT_SUPPORTED;
1198 }
1199
1200 Assert(iBit <= 0x1fff);
1201 if (ASMBitTest(pbMsrBitmap, iBit))
1202 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1203 else
1204 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1205
1206 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1207 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1208 else
1209 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1210 return VINF_SUCCESS;
1211}
1212#endif /* VBOX_STRICT */
1213
1214
1215/**
1216 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1217 * area.
1218 *
1219 * @returns VBox status code.
1220 * @param pVCpu Pointer to the VMCPU.
1221 * @param cMsrs The number of MSRs.
1222 */
1223DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1224{
1225 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1226 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1227 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1228 {
1229 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1230 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1231 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1232 }
1233
1234 /* Update number of guest MSRs to load/store across the world-switch. */
1235 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1239 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1240
1241 /* Update the VCPU's copy of the MSR count. */
1242 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1243
1244 return VINF_SUCCESS;
1245}
1246
1247
1248/**
1249 * Adds a new (or updates the value of an existing) guest/host MSR
1250 * pair to be swapped during the world-switch as part of the
1251 * auto-load/store MSR area in the VMCS.
1252 *
1253 * @returns true if the MSR was added -and- its value was updated, false
1254 * otherwise.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 * @param uGuestMsr Value of the guest MSR.
1258 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1259 * necessary.
1260 */
1261static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1262{
1263 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1264 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1265 uint32_t i;
1266 for (i = 0; i < cMsrs; i++)
1267 {
1268 if (pGuestMsr->u32Msr == uMsr)
1269 break;
1270 pGuestMsr++;
1271 }
1272
1273 bool fAdded = false;
1274 if (i == cMsrs)
1275 {
1276 ++cMsrs;
1277 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1278 AssertRC(rc);
1279
1280 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1281 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1282 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1283
1284 fAdded = true;
1285 }
1286
1287 /* Update the MSR values in the auto-load/store MSR area. */
1288 pGuestMsr->u32Msr = uMsr;
1289 pGuestMsr->u64Value = uGuestMsrValue;
1290
1291 /* Create/update the MSR slot in the host MSR area. */
1292 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1293 pHostMsr += i;
1294 pHostMsr->u32Msr = uMsr;
1295
1296 /*
1297 * Update the host MSR only when requested by the caller AND when we're
1298 * adding it to the auto-load/store area. Otherwise, it would have been
1299 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1300 */
1301 bool fUpdatedMsrValue = false;
1302 if ( fAdded
1303 && fUpdateHostMsr)
1304 {
1305 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1306 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1307 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1308 fUpdatedMsrValue = true;
1309 }
1310
1311 return fUpdatedMsrValue;
1312}
1313
1314
1315/**
1316 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1317 * auto-load/store MSR area in the VMCS.
1318 *
1319 * @returns VBox status code.
1320 * @param pVCpu Pointer to the VMCPU.
1321 * @param uMsr The MSR.
1322 */
1323static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1324{
1325 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1327 for (uint32_t i = 0; i < cMsrs; i++)
1328 {
1329 /* Find the MSR. */
1330 if (pGuestMsr->u32Msr == uMsr)
1331 {
1332 /* If it's the last MSR, simply reduce the count. */
1333 if (i == cMsrs - 1)
1334 {
1335 --cMsrs;
1336 break;
1337 }
1338
1339 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1340 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1341 pLastGuestMsr += cMsrs - 1;
1342 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1343 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1344
1345 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1346 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1347 pLastHostMsr += cMsrs - 1;
1348 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1349 pHostMsr->u64Value = pLastHostMsr->u64Value;
1350 --cMsrs;
1351 break;
1352 }
1353 pGuestMsr++;
1354 }
1355
1356 /* Update the VMCS if the count changed (meaning the MSR was found). */
1357 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1358 {
1359 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1360 AssertRCReturn(rc, rc);
1361
1362 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1363 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1364 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1365
1366 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1367 return VINF_SUCCESS;
1368 }
1369
1370 return VERR_NOT_FOUND;
1371}
1372
1373
1374/**
1375 * Checks if the specified guest MSR is part of the auto-load/store area in
1376 * the VMCS.
1377 *
1378 * @returns true if found, false otherwise.
1379 * @param pVCpu Pointer to the VMCPU.
1380 * @param uMsr The MSR to find.
1381 */
1382static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1383{
1384 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1385 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1386
1387 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1388 {
1389 if (pGuestMsr->u32Msr == uMsr)
1390 return true;
1391 }
1392 return false;
1393}
1394
1395
1396/**
1397 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1398 *
1399 * @param pVCpu Pointer to the VMCPU.
1400 *
1401 * @remarks No-long-jump zone!!!
1402 */
1403static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1404{
1405 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1406 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1407 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1408 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1409
1410 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1411 {
1412 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1413
1414 /*
1415 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1416 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1417 */
1418 if (pHostMsr->u32Msr == MSR_K6_EFER)
1419 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1420 else
1421 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1422 }
1423
1424 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1425}
1426
1427
1428#if HC_ARCH_BITS == 64
1429/**
1430 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1431 * perform lazy restoration of the host MSRs while leaving VT-x.
1432 *
1433 * @param pVCpu Pointer to the VMCPU.
1434 *
1435 * @remarks No-long-jump zone!!!
1436 */
1437static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1438{
1439 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440
1441 /*
1442 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1443 */
1444 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1445 {
1446 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1447 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1448 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1449 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1450 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1451 }
1452}
1453
1454
1455/**
1456 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1457 * lazily while leaving VT-x.
1458 *
1459 * @returns true if it does, false otherwise.
1460 * @param pVCpu Pointer to the VMCPU.
1461 * @param uMsr The MSR to check.
1462 */
1463static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1464{
1465 NOREF(pVCpu);
1466 switch (uMsr)
1467 {
1468 case MSR_K8_LSTAR:
1469 case MSR_K6_STAR:
1470 case MSR_K8_SF_MASK:
1471 case MSR_K8_KERNEL_GS_BASE:
1472 return true;
1473 }
1474 return false;
1475}
1476
1477
1478/**
1479 * Saves a set of guest MSRs back into the guest-CPU context.
1480 *
1481 * @param pVCpu Pointer to the VMCPU.
1482 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1483 * out-of-sync. Make sure to update the required fields
1484 * before using them.
1485 *
1486 * @remarks No-long-jump zone!!!
1487 */
1488static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1489{
1490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1491 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1492
1493 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1494 {
1495 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1496 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1497 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1498 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1499 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1500 }
1501}
1502
1503
1504/**
1505 * Loads a set of guests MSRs to allow read/passthru to the guest.
1506 *
1507 * The name of this function is slightly confusing. This function does NOT
1508 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1509 * common prefix for functions dealing with "lazy restoration" of the shared
1510 * MSRs.
1511 *
1512 * @param pVCpu Pointer to the VMCPU.
1513 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1514 * out-of-sync. Make sure to update the required fields
1515 * before using them.
1516 *
1517 * @remarks No-long-jump zone!!!
1518 */
1519static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1520{
1521 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1522 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1523
1524#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1525 do { \
1526 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1527 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1528 else \
1529 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1530 } while (0)
1531
1532 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1533 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1534 {
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1539 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1540 }
1541 else
1542 {
1543 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1544 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1545 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1546 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1547 }
1548
1549#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1550}
1551
1552
1553/**
1554 * Performs lazy restoration of the set of host MSRs if they were previously
1555 * loaded with guest MSR values.
1556 *
1557 * @param pVCpu Pointer to the VMCPU.
1558 *
1559 * @remarks No-long-jump zone!!!
1560 * @remarks The guest MSRs should have been saved back into the guest-CPU
1561 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1562 */
1563static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1564{
1565 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1566 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1567
1568 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1569 {
1570 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1571 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1572 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1573 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1574 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1575 }
1576 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1577}
1578#endif /* HC_ARCH_BITS == 64 */
1579
1580
1581/**
1582 * Verifies that our cached values of the VMCS controls are all
1583 * consistent with what's actually present in the VMCS.
1584 *
1585 * @returns VBox status code.
1586 * @param pVCpu Pointer to the VMCPU.
1587 */
1588static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1589{
1590 uint32_t u32Val;
1591 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1592 AssertRCReturn(rc, rc);
1593 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1594 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1595
1596 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1597 AssertRCReturn(rc, rc);
1598 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1599 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1600
1601 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1602 AssertRCReturn(rc, rc);
1603 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1604 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1605
1606 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1607 AssertRCReturn(rc, rc);
1608 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1609 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1610
1611 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1612 {
1613 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1614 AssertRCReturn(rc, rc);
1615 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1616 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1617 }
1618
1619 return VINF_SUCCESS;
1620}
1621
1622
1623#ifdef VBOX_STRICT
1624/**
1625 * Verifies that our cached host EFER value has not changed
1626 * since we cached it.
1627 *
1628 * @param pVCpu Pointer to the VMCPU.
1629 */
1630static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1631{
1632 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1633
1634 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1635 {
1636 uint64_t u64Val;
1637 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1638 AssertRC(rc);
1639
1640 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1641 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1642 }
1643}
1644
1645
1646/**
1647 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1648 * VMCS are correct.
1649 *
1650 * @param pVCpu Pointer to the VMCPU.
1651 */
1652static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1653{
1654 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655
1656 /* Verify MSR counts in the VMCS are what we think it should be. */
1657 uint32_t cMsrs;
1658 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1659 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1662 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1665 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1666
1667 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1668 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1669 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1670 {
1671 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1672 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1673 pGuestMsr->u32Msr, cMsrs));
1674
1675 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1676 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1677 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1678
1679 /* Verify that the permissions are as expected in the MSR bitmap. */
1680 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1681 {
1682 VMXMSREXITREAD enmRead;
1683 VMXMSREXITWRITE enmWrite;
1684 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1685 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1686 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1687 {
1688 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1689 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1690 }
1691 else
1692 {
1693 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1694 pGuestMsr->u32Msr, cMsrs));
1695 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1696 pGuestMsr->u32Msr, cMsrs));
1697 }
1698 }
1699 }
1700}
1701#endif /* VBOX_STRICT */
1702
1703
1704/**
1705 * Flushes the TLB using EPT.
1706 *
1707 * @returns VBox status code.
1708 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1709 * enmFlush).
1710 * @param enmFlush Type of flush.
1711 *
1712 * @remarks Caller is responsible for making sure this function is called only
1713 * when NestedPaging is supported and providing @a enmFlush that is
1714 * supported by the CPU.
1715 * @remarks Can be called with interrupts disabled.
1716 */
1717static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1718{
1719 uint64_t au64Descriptor[2];
1720 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1721 au64Descriptor[0] = 0;
1722 else
1723 {
1724 Assert(pVCpu);
1725 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1726 }
1727 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1728
1729 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1730 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1731 rc));
1732 if ( RT_SUCCESS(rc)
1733 && pVCpu)
1734 {
1735 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1736 }
1737}
1738
1739
1740/**
1741 * Flushes the TLB using VPID.
1742 *
1743 * @returns VBox status code.
1744 * @param pVM Pointer to the VM.
1745 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1746 * enmFlush).
1747 * @param enmFlush Type of flush.
1748 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1749 * on @a enmFlush).
1750 *
1751 * @remarks Can be called with interrupts disabled.
1752 */
1753static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1754{
1755 NOREF(pVM);
1756 AssertPtr(pVM);
1757 Assert(pVM->hm.s.vmx.fVpid);
1758
1759 uint64_t au64Descriptor[2];
1760 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1761 {
1762 au64Descriptor[0] = 0;
1763 au64Descriptor[1] = 0;
1764 }
1765 else
1766 {
1767 AssertPtr(pVCpu);
1768 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1769 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1770 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1771 au64Descriptor[1] = GCPtr;
1772 }
1773
1774 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1775 AssertMsg(rc == VINF_SUCCESS,
1776 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1777 if ( RT_SUCCESS(rc)
1778 && pVCpu)
1779 {
1780 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1781 }
1782}
1783
1784
1785/**
1786 * Invalidates a guest page by guest virtual address. Only relevant for
1787 * EPT/VPID, otherwise there is nothing really to invalidate.
1788 *
1789 * @returns VBox status code.
1790 * @param pVM Pointer to the VM.
1791 * @param pVCpu Pointer to the VMCPU.
1792 * @param GCVirt Guest virtual address of the page to invalidate.
1793 */
1794VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1795{
1796 AssertPtr(pVM);
1797 AssertPtr(pVCpu);
1798 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1799
1800 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1801 if (!fFlushPending)
1802 {
1803 /*
1804 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1805 * See @bugref{6043} and @bugref{6177}.
1806 *
1807 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1808 * function maybe called in a loop with individual addresses.
1809 */
1810 if (pVM->hm.s.vmx.fVpid)
1811 {
1812 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1813 {
1814 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1815 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1816 }
1817 else
1818 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1819 }
1820 else if (pVM->hm.s.fNestedPaging)
1821 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1822 }
1823
1824 return VINF_SUCCESS;
1825}
1826
1827
1828/**
1829 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1830 * otherwise there is nothing really to invalidate.
1831 *
1832 * @returns VBox status code.
1833 * @param pVM Pointer to the VM.
1834 * @param pVCpu Pointer to the VMCPU.
1835 * @param GCPhys Guest physical address of the page to invalidate.
1836 */
1837VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1838{
1839 NOREF(pVM); NOREF(GCPhys);
1840 LogFlowFunc(("%RGp\n", GCPhys));
1841
1842 /*
1843 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1844 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1845 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1846 */
1847 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1848 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1849 return VINF_SUCCESS;
1850}
1851
1852
1853/**
1854 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1855 * case where neither EPT nor VPID is supported by the CPU.
1856 *
1857 * @param pVM Pointer to the VM.
1858 * @param pVCpu Pointer to the VMCPU.
1859 * @param pCpu Pointer to the global HM struct.
1860 *
1861 * @remarks Called with interrupts disabled.
1862 */
1863static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1864{
1865 AssertPtr(pVCpu);
1866 AssertPtr(pCpu);
1867 NOREF(pVM);
1868
1869 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1870
1871 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1872#if 0
1873 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1874 pVCpu->hm.s.TlbShootdown.cPages = 0;
1875#endif
1876
1877 Assert(pCpu->idCpu != NIL_RTCPUID);
1878 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1879 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1880 pVCpu->hm.s.fForceTLBFlush = false;
1881 return;
1882}
1883
1884
1885/**
1886 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1887 *
1888 * @param pVM Pointer to the VM.
1889 * @param pVCpu Pointer to the VMCPU.
1890 * @param pCpu Pointer to the global HM CPU struct.
1891 * @remarks All references to "ASID" in this function pertains to "VPID" in
1892 * Intel's nomenclature. The reason is, to avoid confusion in compare
1893 * statements since the host-CPU copies are named "ASID".
1894 *
1895 * @remarks Called with interrupts disabled.
1896 */
1897static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1898{
1899#ifdef VBOX_WITH_STATISTICS
1900 bool fTlbFlushed = false;
1901# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1902# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1903 if (!fTlbFlushed) \
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1905 } while (0)
1906#else
1907# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1908# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1909#endif
1910
1911 AssertPtr(pVM);
1912 AssertPtr(pCpu);
1913 AssertPtr(pVCpu);
1914 Assert(pCpu->idCpu != NIL_RTCPUID);
1915
1916 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1917 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1918 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1919
1920 /*
1921 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1922 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1923 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1924 */
1925 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1926 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1927 {
1928 ++pCpu->uCurrentAsid;
1929 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1930 {
1931 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1932 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1933 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1934 }
1935
1936 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1937 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1938 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1939
1940 /*
1941 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1942 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1943 */
1944 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1945 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946 HMVMX_SET_TAGGED_TLB_FLUSHED();
1947 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1948 }
1949
1950 /* Check for explicit TLB shootdowns. */
1951 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1952 {
1953 /*
1954 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1955 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1956 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1957 * but not guest-physical mappings.
1958 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1959 */
1960 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1962 HMVMX_SET_TAGGED_TLB_FLUSHED();
1963 }
1964
1965 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1966 * where it is commented out. Support individual entry flushing
1967 * someday. */
1968#if 0
1969 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1970 {
1971 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1972
1973 /*
1974 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1975 * as supported by the CPU.
1976 */
1977 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1978 {
1979 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1980 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1981 }
1982 else
1983 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1984
1985 HMVMX_SET_TAGGED_TLB_FLUSHED();
1986 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1987 pVCpu->hm.s.TlbShootdown.cPages = 0;
1988 }
1989#endif
1990
1991 pVCpu->hm.s.fForceTLBFlush = false;
1992
1993 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1994
1995 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1996 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1997 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1998 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1999 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2000 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2001 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2002 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2003 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2004
2005 /* Update VMCS with the VPID. */
2006 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2007 AssertRC(rc);
2008
2009#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2010}
2011
2012
2013/**
2014 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2015 *
2016 * @returns VBox status code.
2017 * @param pVM Pointer to the VM.
2018 * @param pVCpu Pointer to the VMCPU.
2019 * @param pCpu Pointer to the global HM CPU struct.
2020 *
2021 * @remarks Called with interrupts disabled.
2022 */
2023static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2024{
2025 AssertPtr(pVM);
2026 AssertPtr(pVCpu);
2027 AssertPtr(pCpu);
2028 Assert(pCpu->idCpu != NIL_RTCPUID);
2029 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2030 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2031
2032 /*
2033 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2034 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2035 */
2036 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2037 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2041 }
2042
2043 /* Check for explicit TLB shootdown flushes. */
2044 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2045 {
2046 pVCpu->hm.s.fForceTLBFlush = true;
2047 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2048 }
2049
2050 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2051 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2052
2053 if (pVCpu->hm.s.fForceTLBFlush)
2054 {
2055 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2056 pVCpu->hm.s.fForceTLBFlush = false;
2057 }
2058 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2059 * where it is commented out. Support individual entry flushing
2060 * someday. */
2061#if 0
2062 else
2063 {
2064 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2065 {
2066 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2067 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2068 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2069 }
2070 else
2071 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2072
2073 pVCpu->hm.s.TlbShootdown.cPages = 0;
2074 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2075 }
2076#endif
2077}
2078
2079
2080/**
2081 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2082 *
2083 * @returns VBox status code.
2084 * @param pVM Pointer to the VM.
2085 * @param pVCpu Pointer to the VMCPU.
2086 * @param pCpu Pointer to the global HM CPU struct.
2087 *
2088 * @remarks Called with interrupts disabled.
2089 */
2090static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2091{
2092 AssertPtr(pVM);
2093 AssertPtr(pVCpu);
2094 AssertPtr(pCpu);
2095 Assert(pCpu->idCpu != NIL_RTCPUID);
2096 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2097 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2098
2099 /*
2100 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2101 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2102 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2103 */
2104 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2105 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2106 {
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2109 }
2110
2111 /* Check for explicit TLB shootdown flushes. */
2112 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2113 {
2114 /*
2115 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2116 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2117 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2118 */
2119 pVCpu->hm.s.fForceTLBFlush = true;
2120 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2121 }
2122
2123 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2124 if (pVCpu->hm.s.fForceTLBFlush)
2125 {
2126 ++pCpu->uCurrentAsid;
2127 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2128 {
2129 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2130 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2131 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2132 }
2133
2134 pVCpu->hm.s.fForceTLBFlush = false;
2135 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2136 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2137 if (pCpu->fFlushAsidBeforeUse)
2138 {
2139 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2140 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2141 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2142 {
2143 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2144 pCpu->fFlushAsidBeforeUse = false;
2145 }
2146 else
2147 {
2148 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2149 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2150 }
2151 }
2152 }
2153 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2154 * where it is commented out. Support individual entry flushing
2155 * someday. */
2156#if 0
2157 else
2158 {
2159 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2160 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2161 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2162 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2163
2164 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2165 {
2166 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2167 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168 {
2169 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2170 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2171 }
2172 else
2173 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2174
2175 pVCpu->hm.s.TlbShootdown.cPages = 0;
2176 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2177 }
2178 else
2179 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2180 }
2181#endif
2182
2183 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2184 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2185 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2186 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2187 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2188 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2189 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2190
2191 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2192 AssertRC(rc);
2193}
2194
2195
2196/**
2197 * Flushes the guest TLB entry based on CPU capabilities.
2198 *
2199 * @param pVCpu Pointer to the VMCPU.
2200 * @param pCpu Pointer to the global HM CPU struct.
2201 */
2202DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2203{
2204#ifdef HMVMX_ALWAYS_FLUSH_TLB
2205 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2206#endif
2207 PVM pVM = pVCpu->CTX_SUFF(pVM);
2208 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2209 {
2210 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2211 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2212 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2214 default:
2215 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2216 break;
2217 }
2218
2219 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2220 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2221
2222 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2223}
2224
2225
2226/**
2227 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2228 * TLB entries from the host TLB before VM-entry.
2229 *
2230 * @returns VBox status code.
2231 * @param pVM Pointer to the VM.
2232 */
2233static int hmR0VmxSetupTaggedTlb(PVM pVM)
2234{
2235 /*
2236 * Determine optimal flush type for Nested Paging.
2237 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2238 * guest execution (see hmR3InitFinalizeR0()).
2239 */
2240 if (pVM->hm.s.fNestedPaging)
2241 {
2242 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2245 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2246 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2248 else
2249 {
2250 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2251 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254
2255 /* Make sure the write-back cacheable memory type for EPT is supported. */
2256 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2257 {
2258 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2259 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263 else
2264 {
2265 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2266 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2268 }
2269 }
2270
2271 /*
2272 * Determine optimal flush type for VPID.
2273 */
2274 if (pVM->hm.s.vmx.fVpid)
2275 {
2276 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2277 {
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2280 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2282 else
2283 {
2284 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2285 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2286 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2287 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2288 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2289 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2290 pVM->hm.s.vmx.fVpid = false;
2291 }
2292 }
2293 else
2294 {
2295 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2296 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2297 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2298 pVM->hm.s.vmx.fVpid = false;
2299 }
2300 }
2301
2302 /*
2303 * Setup the handler for flushing tagged-TLBs.
2304 */
2305 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2306 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2307 else if (pVM->hm.s.fNestedPaging)
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2309 else if (pVM->hm.s.vmx.fVpid)
2310 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2311 else
2312 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2313 return VINF_SUCCESS;
2314}
2315
2316
2317/**
2318 * Sets up pin-based VM-execution controls in the VMCS.
2319 *
2320 * @returns VBox status code.
2321 * @param pVM Pointer to the VM.
2322 * @param pVCpu Pointer to the VMCPU.
2323 */
2324static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2325{
2326 AssertPtr(pVM);
2327 AssertPtr(pVCpu);
2328
2329 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2330 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2331
2332 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2333 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2334
2335 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2336 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2337
2338 /* Enable the VMX preemption timer. */
2339 if (pVM->hm.s.vmx.fUsePreemptTimer)
2340 {
2341 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2342 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2343 }
2344
2345 if ((val & zap) != val)
2346 {
2347 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2348 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2349 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2350 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2351 }
2352
2353 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2354 AssertRCReturn(rc, rc);
2355
2356 pVCpu->hm.s.vmx.u32PinCtls = val;
2357 return rc;
2358}
2359
2360
2361/**
2362 * Sets up processor-based VM-execution controls in the VMCS.
2363 *
2364 * @returns VBox status code.
2365 * @param pVM Pointer to the VM.
2366 * @param pVMCPU Pointer to the VMCPU.
2367 */
2368static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2369{
2370 AssertPtr(pVM);
2371 AssertPtr(pVCpu);
2372
2373 int rc = VERR_INTERNAL_ERROR_5;
2374 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2375 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2376
2377 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2378 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2384
2385 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2386 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2387 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2388 {
2389 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2390 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2391 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2392 }
2393
2394 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2395 if (!pVM->hm.s.fNestedPaging)
2396 {
2397 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2398 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2399 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2400 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2401 }
2402
2403 /* Use TPR shadowing if supported by the CPU. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2405 {
2406 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2407 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2408 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2409 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2410 AssertRCReturn(rc, rc);
2411
2412 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2413 /* CR8 writes cause a VM-exit based on TPR threshold. */
2414 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2415 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2416 }
2417 else
2418 {
2419 /*
2420 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2421 * Set this control only for 64-bit guests.
2422 */
2423 if (pVM->hm.s.fAllow64BitGuests)
2424 {
2425 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2426 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2427 }
2428 }
2429
2430 /* Use MSR-bitmaps if supported by the CPU. */
2431 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2432 {
2433 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2434
2435 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2436 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2437 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2438 AssertRCReturn(rc, rc);
2439
2440 /*
2441 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2442 * automatically using dedicated fields in the VMCS.
2443 */
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449
2450#if HC_ARCH_BITS == 64
2451 /*
2452 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2453 */
2454 if (pVM->hm.s.fAllow64BitGuests)
2455 {
2456 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 }
2461#endif
2462 }
2463
2464 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2465 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2466 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2467
2468 if ((val & zap) != val)
2469 {
2470 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2471 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2472 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2473 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2474 }
2475
2476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2477 AssertRCReturn(rc, rc);
2478
2479 pVCpu->hm.s.vmx.u32ProcCtls = val;
2480
2481 /*
2482 * Secondary processor-based VM-execution controls.
2483 */
2484 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2485 {
2486 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2487 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2488
2489 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2490 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2491
2492 if (pVM->hm.s.fNestedPaging)
2493 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2494 else
2495 {
2496 /*
2497 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2498 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2499 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2500 */
2501 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2502 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2503 }
2504
2505 if (pVM->hm.s.vmx.fVpid)
2506 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2507
2508 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2509 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2510
2511 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2512 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2513 * done dynamically. */
2514 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2515 {
2516 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2517 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2518 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2519 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2520 AssertRCReturn(rc, rc);
2521 }
2522
2523 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2524 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2525
2526 if ((val & zap) != val)
2527 {
2528 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2529 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2530 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2531 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2532 }
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2535 AssertRCReturn(rc, rc);
2536
2537 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2538 }
2539 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2542 "available\n"));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 return VINF_SUCCESS;
2548}
2549
2550
2551/**
2552 * Sets up miscellaneous (everything other than Pin & Processor-based
2553 * VM-execution) control fields in the VMCS.
2554 *
2555 * @returns VBox status code.
2556 * @param pVM Pointer to the VM.
2557 * @param pVCpu Pointer to the VMCPU.
2558 */
2559static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2560{
2561 NOREF(pVM);
2562 AssertPtr(pVM);
2563 AssertPtr(pVCpu);
2564
2565 int rc = VERR_GENERAL_FAILURE;
2566
2567 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2568#if 0
2569 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2570 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2571 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2572
2573 /*
2574 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2575 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2576 * We thus use the exception bitmap to control it rather than use both.
2577 */
2578 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2580
2581 /** @todo Explore possibility of using IO-bitmaps. */
2582 /* All IO & IOIO instructions cause VM-exits. */
2583 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2584 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2585
2586 /* Initialize the MSR-bitmap area. */
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2590#endif
2591
2592 /* Setup MSR auto-load/store area. */
2593 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2595 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 AssertRCReturn(rc, rc);
2597 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599
2600 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 AssertRCReturn(rc, rc);
2604
2605 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2607 AssertRCReturn(rc, rc);
2608
2609 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2610#if 0
2611 /* Setup debug controls */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2613 AssertRCReturn(rc, rc);
2614 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 return rc;
2619}
2620
2621
2622/**
2623 * Sets up the initial exception bitmap in the VMCS based on static conditions
2624 * (i.e. conditions that cannot ever change after starting the VM).
2625 *
2626 * @returns VBox status code.
2627 * @param pVM Pointer to the VM.
2628 * @param pVCpu Pointer to the VMCPU.
2629 */
2630static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2631{
2632 AssertPtr(pVM);
2633 AssertPtr(pVCpu);
2634
2635 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2636
2637 uint32_t u32XcptBitmap = 0;
2638
2639 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2640 if (!pVM->hm.s.fNestedPaging)
2641 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2642
2643 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2644 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2645 AssertRCReturn(rc, rc);
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial guest-state mask. The guest-state mask is consulted
2652 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2653 * for the nested virtualization case (as it would cause a VM-exit).
2654 *
2655 * @param pVCpu Pointer to the VMCPU.
2656 */
2657static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2658{
2659 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2660 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x initialization.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM Pointer to the VM.
2670 */
2671VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675 int rc = hmR0VmxStructsAlloc(pVM);
2676 if (RT_FAILURE(rc))
2677 {
2678 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2679 return rc;
2680 }
2681
2682 return VINF_SUCCESS;
2683}
2684
2685
2686/**
2687 * Does per-VM VT-x termination.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM Pointer to the VM.
2691 */
2692VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2693{
2694 LogFlowFunc(("pVM=%p\n", pVM));
2695
2696#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2697 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2698 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2699#endif
2700 hmR0VmxStructsFree(pVM);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Sets up the VM for execution under VT-x.
2707 * This function is only called once per-VM during initialization.
2708 *
2709 * @returns VBox status code.
2710 * @param pVM Pointer to the VM.
2711 */
2712VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2713{
2714 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2716
2717 LogFlowFunc(("pVM=%p\n", pVM));
2718
2719 /*
2720 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2721 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2722 */
2723 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2724 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2725 || !pVM->hm.s.vmx.pRealModeTSS))
2726 {
2727 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2728 return VERR_INTERNAL_ERROR;
2729 }
2730
2731#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2732 /*
2733 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2734 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2735 */
2736 if ( pVM->hm.s.fAllow64BitGuests
2737 && !HMVMX_IS_64BIT_HOST_MODE())
2738 {
2739 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2740 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2741 }
2742#endif
2743
2744 /* Initialize these always, see hmR3InitFinalizeR0().*/
2745 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2746 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2747
2748 /* Setup the tagged-TLB flush handlers. */
2749 int rc = hmR0VmxSetupTaggedTlb(pVM);
2750 if (RT_FAILURE(rc))
2751 {
2752 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2753 return rc;
2754 }
2755
2756 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2757 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2758#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2759 if ( HMVMX_IS_64BIT_HOST_MODE()
2760 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2769 {
2770 PVMCPU pVCpu = &pVM->aCpus[i];
2771 AssertPtr(pVCpu);
2772 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2773
2774 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2775 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2776
2777 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2778 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2779 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2780
2781 /* Set revision dword at the beginning of the VMCS structure. */
2782 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2783
2784 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2785 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 /* Load this VMCS as the current VMCS. */
2790 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2792 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2815 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818#endif
2819
2820 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2821 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2822 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2823 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2824
2825 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2826
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2828 }
2829
2830 return VINF_SUCCESS;
2831}
2832
2833
2834/**
2835 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2836 * the VMCS.
2837 *
2838 * @returns VBox status code.
2839 * @param pVM Pointer to the VM.
2840 * @param pVCpu Pointer to the VMCPU.
2841 */
2842DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2843{
2844 NOREF(pVM); NOREF(pVCpu);
2845
2846 RTCCUINTREG uReg = ASMGetCR0();
2847 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2848 AssertRCReturn(rc, rc);
2849
2850#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2851 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2852 if (HMVMX_IS_64BIT_HOST_MODE())
2853 {
2854 uint64_t uRegCR3 = HMR0Get64bitCR3();
2855 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2856 }
2857 else
2858#endif
2859 {
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 }
2863 AssertRCReturn(rc, rc);
2864
2865 uReg = ASMGetCR4();
2866 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2867 AssertRCReturn(rc, rc);
2868 return rc;
2869}
2870
2871
2872#if HC_ARCH_BITS == 64
2873/**
2874 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2875 * requirements. See hmR0VmxSaveHostSegmentRegs().
2876 */
2877# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2878 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2879 { \
2880 bool fValidSelector = true; \
2881 if ((selValue) & X86_SEL_LDT) \
2882 { \
2883 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2884 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2885 } \
2886 if (fValidSelector) \
2887 { \
2888 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2889 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2890 } \
2891 (selValue) = 0; \
2892 }
2893#endif
2894
2895
2896/**
2897 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2898 * the host-state area in the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM Pointer to the VM.
2902 * @param pVCpu Pointer to the VMCPU.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 int rc = VERR_INTERNAL_ERROR_5;
2907
2908#if HC_ARCH_BITS == 64
2909 /*
2910 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2911 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2912 */
2913 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2914 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2915#endif
2916
2917 /*
2918 * Host DS, ES, FS and GS segment registers.
2919 */
2920#if HC_ARCH_BITS == 64
2921 RTSEL uSelDS = ASMGetDS();
2922 RTSEL uSelES = ASMGetES();
2923 RTSEL uSelFS = ASMGetFS();
2924 RTSEL uSelGS = ASMGetGS();
2925#else
2926 RTSEL uSelDS = 0;
2927 RTSEL uSelES = 0;
2928 RTSEL uSelFS = 0;
2929 RTSEL uSelGS = 0;
2930#endif
2931
2932 /* Recalculate which host-state bits need to be manually restored. */
2933 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2934
2935 /*
2936 * Host CS and SS segment registers.
2937 */
2938#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939 RTSEL uSelCS;
2940 RTSEL uSelSS;
2941 if (HMVMX_IS_64BIT_HOST_MODE())
2942 {
2943 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2944 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2945 }
2946 else
2947 {
2948 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2949 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2950 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2951 }
2952#else
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955#endif
2956
2957 /*
2958 * Host TR segment register.
2959 */
2960 RTSEL uSelTR = ASMGetTR();
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2965 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2966 */
2967 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2968 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2969 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2970 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2971# undef VMXLOCAL_ADJUST_HOST_SEG
2972#endif
2973
2974 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2975 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2976 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2977 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2978 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2979 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2980 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2981 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2982 Assert(uSelCS);
2983 Assert(uSelTR);
2984
2985 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2986#if 0
2987 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2988 Assert(uSelSS != 0);
2989#endif
2990
2991 /* Write these host selector fields into the host-state area in the VMCS. */
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2994#if HC_ARCH_BITS == 64
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2999#endif
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3001
3002 /*
3003 * Host GDTR and IDTR.
3004 */
3005 RTGDTR Gdtr;
3006 RT_ZERO(Gdtr);
3007#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 X86XDTR64 Gdtr64;
3011 X86XDTR64 Idtr64;
3012 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3015
3016 Gdtr.cbGdt = Gdtr64.cb;
3017 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3018 }
3019 else
3020#endif
3021 {
3022 RTIDTR Idtr;
3023 ASMGetGDTR(&Gdtr);
3024 ASMGetIDTR(&Idtr);
3025 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3027
3028#if HC_ARCH_BITS == 64
3029 /*
3030 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3031 * maximum limit (0xffff) on every VM-exit.
3032 */
3033 if (Gdtr.cbGdt != 0xffff)
3034 {
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3036 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3037 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3038 }
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3042 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3043 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3044 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3045 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3046 * hosts where we are pretty sure it won't cause trouble.
3047 */
3048# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3049 if (Idtr.cbIdt < 0x0fff)
3050# else
3051 if (Idtr.cbIdt != 0xffff)
3052# endif
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3055 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3056 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3057 }
3058#endif
3059 }
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3063 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3064 */
3065 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3066 {
3067 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3068 return VERR_VMX_INVALID_HOST_STATE;
3069 }
3070
3071 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3072#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3073 if (HMVMX_IS_64BIT_HOST_MODE())
3074 {
3075 /* We need the 64-bit TR base for hybrid darwin. */
3076 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3077 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3078 }
3079 else
3080#endif
3081 {
3082 uintptr_t uTRBase;
3083#if HC_ARCH_BITS == 64
3084 uTRBase = X86DESC64_BASE(pDesc);
3085
3086 /*
3087 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3088 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3089 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3090 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3091 *
3092 * [1] See Intel spec. 3.5 "System Descriptor Types".
3093 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3094 */
3095 Assert(pDesc->System.u4Type == 11);
3096 if ( pDesc->System.u16LimitLow != 0x67
3097 || pDesc->System.u4LimitHigh)
3098 {
3099 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3100 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3101 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3102 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3103 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3104
3105 /* Store the GDTR here as we need it while restoring TR. */
3106 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3107 }
3108#else
3109 uTRBase = X86DESC_BASE(pDesc);
3110#endif
3111 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3112 }
3113 AssertRCReturn(rc, rc);
3114
3115 /*
3116 * Host FS base and GS base.
3117 */
3118#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3119 if (HMVMX_IS_64BIT_HOST_MODE())
3120 {
3121 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3122 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3123 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3124 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3125
3126# if HC_ARCH_BITS == 64
3127 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3130 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3131 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3132# endif
3133 }
3134#endif
3135 return rc;
3136}
3137
3138
3139/**
3140 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3141 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3142 * the host after every successful VM-exit.
3143 *
3144 * @returns VBox status code.
3145 * @param pVM Pointer to the VM.
3146 * @param pVCpu Pointer to the VMCPU.
3147 *
3148 * @remarks No-long-jump zone!!!
3149 */
3150DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3151{
3152 NOREF(pVM);
3153
3154 AssertPtr(pVCpu);
3155 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3156
3157 int rc = VINF_SUCCESS;
3158#if HC_ARCH_BITS == 64
3159 if (pVM->hm.s.fAllow64BitGuests)
3160 hmR0VmxLazySaveHostMsrs(pVCpu);
3161#endif
3162
3163 /*
3164 * Host Sysenter MSRs.
3165 */
3166 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3167 AssertRCReturn(rc, rc);
3168#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3169 if (HMVMX_IS_64BIT_HOST_MODE())
3170 {
3171 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3172 AssertRCReturn(rc, rc);
3173 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3174 }
3175 else
3176 {
3177 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3178 AssertRCReturn(rc, rc);
3179 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3180 }
3181#elif HC_ARCH_BITS == 32
3182 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3183 AssertRCReturn(rc, rc);
3184 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3185#else
3186 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3187 AssertRCReturn(rc, rc);
3188 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3189#endif
3190 AssertRCReturn(rc, rc);
3191
3192 /*
3193 * Host EFER MSR.
3194 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3195 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3196 */
3197 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3198 {
3199 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3200 AssertRCReturn(rc, rc);
3201 }
3202
3203 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3204 * hmR0VmxLoadGuestExitCtls() !! */
3205
3206 return rc;
3207}
3208
3209
3210/**
3211 * Figures out if we need to swap the EFER MSR which is
3212 * particularly expensive.
3213 *
3214 * We check all relevant bits. For now, that's everything
3215 * besides LMA/LME, as these two bits are handled by VM-entry,
3216 * see hmR0VmxLoadGuestExitCtls() and
3217 * hmR0VMxLoadGuestEntryCtls().
3218 *
3219 * @returns true if we need to load guest EFER, false otherwise.
3220 * @param pVCpu Pointer to the VMCPU.
3221 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3222 * out-of-sync. Make sure to update the required fields
3223 * before using them.
3224 *
3225 * @remarks Requires EFER, CR4.
3226 * @remarks No-long-jump zone!!!
3227 */
3228static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3229{
3230#ifdef HMVMX_ALWAYS_SWAP_EFER
3231 return true;
3232#endif
3233
3234#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3235 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3236 if (CPUMIsGuestInLongMode(pVCpu))
3237 return false;
3238#endif
3239
3240 PVM pVM = pVCpu->CTX_SUFF(pVM);
3241 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3242 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3243
3244 /*
3245 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3246 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3247 */
3248 if ( CPUMIsGuestInLongMode(pVCpu)
3249 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3250 {
3251 return true;
3252 }
3253
3254 /*
3255 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3256 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3257 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3258 */
3259 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3260 && (pMixedCtx->cr0 & X86_CR0_PG)
3261 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3262 {
3263 /* Assert that host is PAE capable. */
3264 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3265 return true;
3266 }
3267
3268 /** @todo Check the latest Intel spec. for any other bits,
3269 * like SMEP/SMAP? */
3270 return false;
3271}
3272
3273
3274/**
3275 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3276 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3277 * controls".
3278 *
3279 * @returns VBox status code.
3280 * @param pVCpu Pointer to the VMCPU.
3281 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3282 * out-of-sync. Make sure to update the required fields
3283 * before using them.
3284 *
3285 * @remarks Requires EFER.
3286 * @remarks No-long-jump zone!!!
3287 */
3288DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3289{
3290 int rc = VINF_SUCCESS;
3291 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3292 {
3293 PVM pVM = pVCpu->CTX_SUFF(pVM);
3294 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3295 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3296
3297 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3298 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3299
3300 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3301 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3302 {
3303 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3304 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3305 }
3306 else
3307 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3308
3309 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3310 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3311 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3312 {
3313 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3314 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3315 }
3316
3317 /*
3318 * The following should -not- be set (since we're not in SMM mode):
3319 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3320 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3321 */
3322
3323 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3324 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3325
3326 if ((val & zap) != val)
3327 {
3328 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3329 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3330 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3331 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3332 }
3333
3334 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3335 AssertRCReturn(rc, rc);
3336
3337 pVCpu->hm.s.vmx.u32EntryCtls = val;
3338 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3339 }
3340 return rc;
3341}
3342
3343
3344/**
3345 * Sets up the VM-exit controls in the VMCS.
3346 *
3347 * @returns VBox status code.
3348 * @param pVM Pointer to the VM.
3349 * @param pVCpu Pointer to the VMCPU.
3350 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3351 * out-of-sync. Make sure to update the required fields
3352 * before using them.
3353 *
3354 * @remarks Requires EFER.
3355 */
3356DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3357{
3358 NOREF(pMixedCtx);
3359
3360 int rc = VINF_SUCCESS;
3361 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3362 {
3363 PVM pVM = pVCpu->CTX_SUFF(pVM);
3364 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3365 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3366
3367 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3368 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3369
3370 /*
3371 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3372 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3373 */
3374#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3375 if (HMVMX_IS_64BIT_HOST_MODE())
3376 {
3377 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3378 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3379 }
3380 else
3381 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3382#else
3383 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3384 {
3385 /* The switcher returns to long mode, EFER is managed by the switcher. */
3386 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3387 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3388 }
3389 else
3390 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3391#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3392
3393 /* If the newer VMCS fields for managing EFER exists, use it. */
3394 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3395 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3396 {
3397 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3398 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3399 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3400 }
3401
3402 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3403 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3404
3405 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3406 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3407 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3408
3409 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3410 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3411
3412 if ((val & zap) != val)
3413 {
3414 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3415 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3416 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3417 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3418 }
3419
3420 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3421 AssertRCReturn(rc, rc);
3422
3423 pVCpu->hm.s.vmx.u32ExitCtls = val;
3424 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3425 }
3426 return rc;
3427}
3428
3429
3430/**
3431 * Loads the guest APIC and related state.
3432 *
3433 * @returns VBox status code.
3434 * @param pVM Pointer to the VM.
3435 * @param pVCpu Pointer to the VMCPU.
3436 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3437 * out-of-sync. Make sure to update the required fields
3438 * before using them.
3439 */
3440DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3441{
3442 NOREF(pMixedCtx);
3443
3444 int rc = VINF_SUCCESS;
3445 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3446 {
3447 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3448 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3449 {
3450 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3451
3452 bool fPendingIntr = false;
3453 uint8_t u8Tpr = 0;
3454 uint8_t u8PendingIntr = 0;
3455 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3456 AssertRCReturn(rc, rc);
3457
3458 /*
3459 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3460 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3461 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3462 * the interrupt when we VM-exit for other reasons.
3463 */
3464 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3465 uint32_t u32TprThreshold = 0;
3466 if (fPendingIntr)
3467 {
3468 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3469 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3470 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3471 if (u8PendingPriority <= u8TprPriority)
3472 u32TprThreshold = u8PendingPriority;
3473 else
3474 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3475 }
3476 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3477
3478 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3479 AssertRCReturn(rc, rc);
3480 }
3481
3482 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3483 }
3484 return rc;
3485}
3486
3487
3488/**
3489 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3490 *
3491 * @returns Guest's interruptibility-state.
3492 * @param pVCpu Pointer to the VMCPU.
3493 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3494 * out-of-sync. Make sure to update the required fields
3495 * before using them.
3496 *
3497 * @remarks No-long-jump zone!!!
3498 */
3499DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3500{
3501 /*
3502 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3503 */
3504 uint32_t uIntrState = 0;
3505 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3506 {
3507 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3508 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3509 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3510 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3511 {
3512 if (pMixedCtx->eflags.Bits.u1IF)
3513 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3514 else
3515 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3516 }
3517 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3518 }
3519
3520 /*
3521 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3522 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3523 * setting this would block host-NMIs and IRET will not clear the blocking.
3524 *
3525 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3526 */
3527 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3528 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3529 {
3530 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3531 }
3532
3533 return uIntrState;
3534}
3535
3536
3537/**
3538 * Loads the guest's interruptibility-state into the guest-state area in the
3539 * VMCS.
3540 *
3541 * @returns VBox status code.
3542 * @param pVCpu Pointer to the VMCPU.
3543 * @param uIntrState The interruptibility-state to set.
3544 */
3545static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3546{
3547 NOREF(pVCpu);
3548 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3549 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3550 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3551 AssertRCReturn(rc, rc);
3552 return rc;
3553}
3554
3555
3556/**
3557 * Loads the guest's RIP into the guest-state area in the VMCS.
3558 *
3559 * @returns VBox status code.
3560 * @param pVCpu Pointer to the VMCPU.
3561 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3562 * out-of-sync. Make sure to update the required fields
3563 * before using them.
3564 *
3565 * @remarks No-long-jump zone!!!
3566 */
3567static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3568{
3569 int rc = VINF_SUCCESS;
3570 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3571 {
3572 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3573 AssertRCReturn(rc, rc);
3574
3575 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3576 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3577 HMCPU_CF_VALUE(pVCpu)));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Loads the guest's RSP into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu Pointer to the VMCPU.
3588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589 * out-of-sync. Make sure to update the required fields
3590 * before using them.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3595{
3596 int rc = VINF_SUCCESS;
3597 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3598 {
3599 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3600 AssertRCReturn(rc, rc);
3601
3602 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3603 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3604 }
3605 return rc;
3606}
3607
3608
3609/**
3610 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3611 *
3612 * @returns VBox status code.
3613 * @param pVCpu Pointer to the VMCPU.
3614 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3615 * out-of-sync. Make sure to update the required fields
3616 * before using them.
3617 *
3618 * @remarks No-long-jump zone!!!
3619 */
3620static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3621{
3622 int rc = VINF_SUCCESS;
3623 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3624 {
3625 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3626 Let us assert it as such and use 32-bit VMWRITE. */
3627 Assert(!(pMixedCtx->rflags.u64 >> 32));
3628 X86EFLAGS Eflags = pMixedCtx->eflags;
3629 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3630 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3631
3632 /*
3633 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3634 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3635 */
3636 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3637 {
3638 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3639 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3640 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3641 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3642 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3643 }
3644
3645 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3646 AssertRCReturn(rc, rc);
3647
3648 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3649 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3650 }
3651 return rc;
3652}
3653
3654
3655/**
3656 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3657 *
3658 * @returns VBox status code.
3659 * @param pVCpu Pointer to the VMCPU.
3660 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3661 * out-of-sync. Make sure to update the required fields
3662 * before using them.
3663 *
3664 * @remarks No-long-jump zone!!!
3665 */
3666DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3667{
3668 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3669 AssertRCReturn(rc, rc);
3670 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3671 AssertRCReturn(rc, rc);
3672 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3673 AssertRCReturn(rc, rc);
3674 return rc;
3675}
3676
3677
3678/**
3679 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3680 * CR0 is partially shared with the host and we have to consider the FPU bits.
3681 *
3682 * @returns VBox status code.
3683 * @param pVM Pointer to the VM.
3684 * @param pVCpu Pointer to the VMCPU.
3685 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3686 * out-of-sync. Make sure to update the required fields
3687 * before using them.
3688 *
3689 * @remarks No-long-jump zone!!!
3690 */
3691static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3692{
3693 /*
3694 * Guest CR0.
3695 * Guest FPU.
3696 */
3697 int rc = VINF_SUCCESS;
3698 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3699 {
3700 Assert(!(pMixedCtx->cr0 >> 32));
3701 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3702 PVM pVM = pVCpu->CTX_SUFF(pVM);
3703
3704 /* The guest's view (read access) of its CR0 is unblemished. */
3705 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3706 AssertRCReturn(rc, rc);
3707 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3708
3709 /* Setup VT-x's view of the guest CR0. */
3710 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3711 if (pVM->hm.s.fNestedPaging)
3712 {
3713 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3714 {
3715 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3716 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3717 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3718 }
3719 else
3720 {
3721 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3722 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3723 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3724 }
3725
3726 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3727 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3728 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729
3730 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3731 AssertRCReturn(rc, rc);
3732 }
3733 else
3734 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3735
3736 /*
3737 * Guest FPU bits.
3738 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3739 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3740 */
3741 u32GuestCR0 |= X86_CR0_NE;
3742 bool fInterceptNM = false;
3743 if (CPUMIsGuestFPUStateActive(pVCpu))
3744 {
3745 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3746 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3747 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3748 }
3749 else
3750 {
3751 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3752 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3753 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3754 }
3755
3756 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3757 bool fInterceptMF = false;
3758 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3759 fInterceptMF = true;
3760
3761 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3762 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3763 {
3764 Assert(PDMVmmDevHeapIsEnabled(pVM));
3765 Assert(pVM->hm.s.vmx.pRealModeTSS);
3766 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3767 fInterceptNM = true;
3768 fInterceptMF = true;
3769 }
3770 else
3771 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3772
3773 if (fInterceptNM)
3774 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3775 else
3776 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3777
3778 if (fInterceptMF)
3779 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3780 else
3781 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3782
3783 /* Additional intercepts for debugging, define these yourself explicitly. */
3784#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3785 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3786 | RT_BIT(X86_XCPT_BP)
3787 | RT_BIT(X86_XCPT_DB)
3788 | RT_BIT(X86_XCPT_DE)
3789 | RT_BIT(X86_XCPT_NM)
3790 | RT_BIT(X86_XCPT_TS)
3791 | RT_BIT(X86_XCPT_UD)
3792 | RT_BIT(X86_XCPT_NP)
3793 | RT_BIT(X86_XCPT_SS)
3794 | RT_BIT(X86_XCPT_GP)
3795 | RT_BIT(X86_XCPT_PF)
3796 | RT_BIT(X86_XCPT_MF)
3797 ;
3798#elif defined(HMVMX_ALWAYS_TRAP_PF)
3799 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3800#endif
3801
3802 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3803
3804 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3805 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3806 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3807 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3808 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3809 else
3810 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3811
3812 u32GuestCR0 |= uSetCR0;
3813 u32GuestCR0 &= uZapCR0;
3814 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3815
3816 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3817 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3818 AssertRCReturn(rc, rc);
3819 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3820 AssertRCReturn(rc, rc);
3821 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3822 uZapCR0));
3823
3824 /*
3825 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3826 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3827 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3828 */
3829 uint32_t u32CR0Mask = 0;
3830 u32CR0Mask = X86_CR0_PE
3831 | X86_CR0_NE
3832 | X86_CR0_WP
3833 | X86_CR0_PG
3834 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3835 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3836 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3837
3838 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3839 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3840 * and @bugref{6944}. */
3841#if 0
3842 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3843 u32CR0Mask &= ~X86_CR0_PE;
3844#endif
3845 if (pVM->hm.s.fNestedPaging)
3846 u32CR0Mask &= ~X86_CR0_WP;
3847
3848 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3849 if (fInterceptNM)
3850 {
3851 u32CR0Mask |= X86_CR0_TS
3852 | X86_CR0_MP;
3853 }
3854
3855 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3856 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3857 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3858 AssertRCReturn(rc, rc);
3859 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3860
3861 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3862 }
3863 return rc;
3864}
3865
3866
3867/**
3868 * Loads the guest control registers (CR3, CR4) into the guest-state area
3869 * in the VMCS.
3870 *
3871 * @returns VBox status code.
3872 * @param pVM Pointer to the VM.
3873 * @param pVCpu Pointer to the VMCPU.
3874 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3875 * out-of-sync. Make sure to update the required fields
3876 * before using them.
3877 *
3878 * @remarks No-long-jump zone!!!
3879 */
3880static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3881{
3882 int rc = VINF_SUCCESS;
3883 PVM pVM = pVCpu->CTX_SUFF(pVM);
3884
3885 /*
3886 * Guest CR2.
3887 * It's always loaded in the assembler code. Nothing to do here.
3888 */
3889
3890 /*
3891 * Guest CR3.
3892 */
3893 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3894 {
3895 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3896 if (pVM->hm.s.fNestedPaging)
3897 {
3898 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3899
3900 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3901 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3902 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3903 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3904
3905 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3906 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3907 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3908
3909 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3910 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3911 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3912 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3913
3914 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3915 AssertRCReturn(rc, rc);
3916 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3917
3918 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3919 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3920 {
3921 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3922 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3923 {
3924 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3925 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3926 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3927 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3928 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3929 }
3930
3931 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3932 have Unrestricted Execution to handle the guest when it's not using paging. */
3933 GCPhysGuestCR3 = pMixedCtx->cr3;
3934 }
3935 else
3936 {
3937 /*
3938 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3939 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3940 * EPT takes care of translating it to host-physical addresses.
3941 */
3942 RTGCPHYS GCPhys;
3943 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3944 Assert(PDMVmmDevHeapIsEnabled(pVM));
3945
3946 /* We obtain it here every time as the guest could have relocated this PCI region. */
3947 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3948 AssertRCReturn(rc, rc);
3949
3950 GCPhysGuestCR3 = GCPhys;
3951 }
3952
3953 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3954 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3955 }
3956 else
3957 {
3958 /* Non-nested paging case, just use the hypervisor's CR3. */
3959 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3960
3961 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3962 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3963 }
3964 AssertRCReturn(rc, rc);
3965
3966 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3967 }
3968
3969 /*
3970 * Guest CR4.
3971 */
3972 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3973 {
3974 Assert(!(pMixedCtx->cr4 >> 32));
3975 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3976
3977 /* The guest's view of its CR4 is unblemished. */
3978 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3979 AssertRCReturn(rc, rc);
3980 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3981
3982 /* Setup VT-x's view of the guest CR4. */
3983 /*
3984 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3985 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3986 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3987 */
3988 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3989 {
3990 Assert(pVM->hm.s.vmx.pRealModeTSS);
3991 Assert(PDMVmmDevHeapIsEnabled(pVM));
3992 u32GuestCR4 &= ~X86_CR4_VME;
3993 }
3994
3995 if (pVM->hm.s.fNestedPaging)
3996 {
3997 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3998 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3999 {
4000 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4001 u32GuestCR4 |= X86_CR4_PSE;
4002 /* Our identity mapping is a 32-bit page directory. */
4003 u32GuestCR4 &= ~X86_CR4_PAE;
4004 }
4005 /* else use guest CR4.*/
4006 }
4007 else
4008 {
4009 /*
4010 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4011 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4012 */
4013 switch (pVCpu->hm.s.enmShadowMode)
4014 {
4015 case PGMMODE_REAL: /* Real-mode. */
4016 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4017 case PGMMODE_32_BIT: /* 32-bit paging. */
4018 {
4019 u32GuestCR4 &= ~X86_CR4_PAE;
4020 break;
4021 }
4022
4023 case PGMMODE_PAE: /* PAE paging. */
4024 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4025 {
4026 u32GuestCR4 |= X86_CR4_PAE;
4027 break;
4028 }
4029
4030 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4031 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4032#ifdef VBOX_ENABLE_64_BITS_GUESTS
4033 break;
4034#endif
4035 default:
4036 AssertFailed();
4037 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4038 }
4039 }
4040
4041 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4042 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4043 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4044 u32GuestCR4 |= uSetCR4;
4045 u32GuestCR4 &= uZapCR4;
4046
4047 /* Write VT-x's view of the guest CR4 into the VMCS. */
4048 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4049 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4050 AssertRCReturn(rc, rc);
4051
4052 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4053 uint32_t u32CR4Mask = 0;
4054 u32CR4Mask = X86_CR4_VME
4055 | X86_CR4_PAE
4056 | X86_CR4_PGE
4057 | X86_CR4_PSE
4058 | X86_CR4_VMXE;
4059 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4060 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4061 AssertRCReturn(rc, rc);
4062
4063 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4064 }
4065 return rc;
4066}
4067
4068
4069/**
4070 * Loads the guest debug registers into the guest-state area in the VMCS.
4071 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4072 *
4073 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4074 *
4075 * @returns VBox status code.
4076 * @param pVCpu Pointer to the VMCPU.
4077 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4078 * out-of-sync. Make sure to update the required fields
4079 * before using them.
4080 *
4081 * @remarks No-long-jump zone!!!
4082 */
4083static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4084{
4085 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4086 return VINF_SUCCESS;
4087
4088#ifdef VBOX_STRICT
4089 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4090 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4091 {
4092 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4093 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4094 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4095 }
4096#endif
4097
4098 int rc;
4099 PVM pVM = pVCpu->CTX_SUFF(pVM);
4100 bool fInterceptDB = false;
4101 bool fInterceptMovDRx = false;
4102 if ( pVCpu->hm.s.fSingleInstruction
4103 || DBGFIsStepping(pVCpu))
4104 {
4105 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4106 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4107 {
4108 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4109 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4110 AssertRCReturn(rc, rc);
4111 Assert(fInterceptDB == false);
4112 }
4113 else
4114 {
4115 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4116 pVCpu->hm.s.fClearTrapFlag = true;
4117 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4118 fInterceptDB = true;
4119 }
4120 }
4121
4122 if ( fInterceptDB
4123 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4124 {
4125 /*
4126 * Use the combined guest and host DRx values found in the hypervisor
4127 * register set because the debugger has breakpoints active or someone
4128 * is single stepping on the host side without a monitor trap flag.
4129 *
4130 * Note! DBGF expects a clean DR6 state before executing guest code.
4131 */
4132#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4133 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4134 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4135 {
4136 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4137 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4138 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4139 }
4140 else
4141#endif
4142 if (!CPUMIsHyperDebugStateActive(pVCpu))
4143 {
4144 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4145 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4146 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4147 }
4148
4149 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4150 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4151 AssertRCReturn(rc, rc);
4152
4153 pVCpu->hm.s.fUsingHyperDR7 = true;
4154 fInterceptDB = true;
4155 fInterceptMovDRx = true;
4156 }
4157 else
4158 {
4159 /*
4160 * If the guest has enabled debug registers, we need to load them prior to
4161 * executing guest code so they'll trigger at the right time.
4162 */
4163 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4164 {
4165#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4166 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4167 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4168 {
4169 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4170 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4171 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4172 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4173 }
4174 else
4175#endif
4176 if (!CPUMIsGuestDebugStateActive(pVCpu))
4177 {
4178 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4179 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4180 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4181 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4182 }
4183 Assert(!fInterceptDB);
4184 Assert(!fInterceptMovDRx);
4185 }
4186 /*
4187 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4188 * must intercept #DB in order to maintain a correct DR6 guest value.
4189 */
4190#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4191 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4192 && !CPUMIsGuestDebugStateActive(pVCpu))
4193#else
4194 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4195#endif
4196 {
4197 fInterceptMovDRx = true;
4198 fInterceptDB = true;
4199 }
4200
4201 /* Update guest DR7. */
4202 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4203 AssertRCReturn(rc, rc);
4204
4205 pVCpu->hm.s.fUsingHyperDR7 = false;
4206 }
4207
4208 /*
4209 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4210 */
4211 if (fInterceptDB)
4212 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4213 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4214 {
4215#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4216 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4217#endif
4218 }
4219 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4220 AssertRCReturn(rc, rc);
4221
4222 /*
4223 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4224 */
4225 if (fInterceptMovDRx)
4226 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4227 else
4228 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4229 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4230 AssertRCReturn(rc, rc);
4231
4232 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4233 return VINF_SUCCESS;
4234}
4235
4236
4237#ifdef VBOX_STRICT
4238/**
4239 * Strict function to validate segment registers.
4240 *
4241 * @remarks ASSUMES CR0 is up to date.
4242 */
4243static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4244{
4245 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4246 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4247 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4248 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4249 && ( !CPUMIsGuestInRealModeEx(pCtx)
4250 && !CPUMIsGuestInV86ModeEx(pCtx)))
4251 {
4252 /* Protected mode checks */
4253 /* CS */
4254 Assert(pCtx->cs.Attr.n.u1Present);
4255 Assert(!(pCtx->cs.Attr.u & 0xf00));
4256 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4257 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4258 || !(pCtx->cs.Attr.n.u1Granularity));
4259 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4260 || (pCtx->cs.Attr.n.u1Granularity));
4261 /* CS cannot be loaded with NULL in protected mode. */
4262 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4263 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4264 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4265 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4266 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4267 else
4268 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4269 /* SS */
4270 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4271 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4272 if ( !(pCtx->cr0 & X86_CR0_PE)
4273 || pCtx->cs.Attr.n.u4Type == 3)
4274 {
4275 Assert(!pCtx->ss.Attr.n.u2Dpl);
4276 }
4277 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4278 {
4279 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4280 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4281 Assert(pCtx->ss.Attr.n.u1Present);
4282 Assert(!(pCtx->ss.Attr.u & 0xf00));
4283 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4284 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4285 || !(pCtx->ss.Attr.n.u1Granularity));
4286 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4287 || (pCtx->ss.Attr.n.u1Granularity));
4288 }
4289 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4290 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4291 {
4292 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4293 Assert(pCtx->ds.Attr.n.u1Present);
4294 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4295 Assert(!(pCtx->ds.Attr.u & 0xf00));
4296 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4297 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4298 || !(pCtx->ds.Attr.n.u1Granularity));
4299 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4300 || (pCtx->ds.Attr.n.u1Granularity));
4301 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4302 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4303 }
4304 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4305 {
4306 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4307 Assert(pCtx->es.Attr.n.u1Present);
4308 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4309 Assert(!(pCtx->es.Attr.u & 0xf00));
4310 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4311 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4312 || !(pCtx->es.Attr.n.u1Granularity));
4313 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4314 || (pCtx->es.Attr.n.u1Granularity));
4315 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4316 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4317 }
4318 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4319 {
4320 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4321 Assert(pCtx->fs.Attr.n.u1Present);
4322 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4323 Assert(!(pCtx->fs.Attr.u & 0xf00));
4324 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4325 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4326 || !(pCtx->fs.Attr.n.u1Granularity));
4327 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4328 || (pCtx->fs.Attr.n.u1Granularity));
4329 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4330 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4331 }
4332 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4333 {
4334 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4335 Assert(pCtx->gs.Attr.n.u1Present);
4336 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4337 Assert(!(pCtx->gs.Attr.u & 0xf00));
4338 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4339 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4340 || !(pCtx->gs.Attr.n.u1Granularity));
4341 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4342 || (pCtx->gs.Attr.n.u1Granularity));
4343 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4344 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4345 }
4346 /* 64-bit capable CPUs. */
4347# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4348 if (HMVMX_IS_64BIT_HOST_MODE())
4349 {
4350 Assert(!(pCtx->cs.u64Base >> 32));
4351 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4352 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4353 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4354 }
4355# endif
4356 }
4357 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4358 || ( CPUMIsGuestInRealModeEx(pCtx)
4359 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4360 {
4361 /* Real and v86 mode checks. */
4362 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4363 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4364 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4365 {
4366 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4367 }
4368 else
4369 {
4370 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4371 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4372 }
4373
4374 /* CS */
4375 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4376 Assert(pCtx->cs.u32Limit == 0xffff);
4377 Assert(u32CSAttr == 0xf3);
4378 /* SS */
4379 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4380 Assert(pCtx->ss.u32Limit == 0xffff);
4381 Assert(u32SSAttr == 0xf3);
4382 /* DS */
4383 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4384 Assert(pCtx->ds.u32Limit == 0xffff);
4385 Assert(u32DSAttr == 0xf3);
4386 /* ES */
4387 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4388 Assert(pCtx->es.u32Limit == 0xffff);
4389 Assert(u32ESAttr == 0xf3);
4390 /* FS */
4391 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4392 Assert(pCtx->fs.u32Limit == 0xffff);
4393 Assert(u32FSAttr == 0xf3);
4394 /* GS */
4395 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4396 Assert(pCtx->gs.u32Limit == 0xffff);
4397 Assert(u32GSAttr == 0xf3);
4398 /* 64-bit capable CPUs. */
4399# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4400 if (HMVMX_IS_64BIT_HOST_MODE())
4401 {
4402 Assert(!(pCtx->cs.u64Base >> 32));
4403 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4404 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4405 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4406 }
4407# endif
4408 }
4409}
4410#endif /* VBOX_STRICT */
4411
4412
4413/**
4414 * Writes a guest segment register into the guest-state area in the VMCS.
4415 *
4416 * @returns VBox status code.
4417 * @param pVCpu Pointer to the VMCPU.
4418 * @param idxSel Index of the selector in the VMCS.
4419 * @param idxLimit Index of the segment limit in the VMCS.
4420 * @param idxBase Index of the segment base in the VMCS.
4421 * @param idxAccess Index of the access rights of the segment in the VMCS.
4422 * @param pSelReg Pointer to the segment selector.
4423 *
4424 * @remarks No-long-jump zone!!!
4425 */
4426static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4427 uint32_t idxAccess, PCPUMSELREG pSelReg)
4428{
4429 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4430 AssertRCReturn(rc, rc);
4431 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4432 AssertRCReturn(rc, rc);
4433 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4434 AssertRCReturn(rc, rc);
4435
4436 uint32_t u32Access = pSelReg->Attr.u;
4437 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4438 {
4439 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4440 u32Access = 0xf3;
4441 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4442 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4443 }
4444 else
4445 {
4446 /*
4447 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4448 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4449 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4450 * loaded in protected-mode have their attribute as 0.
4451 */
4452 if (!u32Access)
4453 u32Access = X86DESCATTR_UNUSABLE;
4454 }
4455
4456 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4457 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4458 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4459
4460 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4461 AssertRCReturn(rc, rc);
4462 return rc;
4463}
4464
4465
4466/**
4467 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4468 * into the guest-state area in the VMCS.
4469 *
4470 * @returns VBox status code.
4471 * @param pVM Pointer to the VM.
4472 * @param pVCPU Pointer to the VMCPU.
4473 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4474 * out-of-sync. Make sure to update the required fields
4475 * before using them.
4476 *
4477 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4478 * @remarks No-long-jump zone!!!
4479 */
4480static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4481{
4482 int rc = VERR_INTERNAL_ERROR_5;
4483 PVM pVM = pVCpu->CTX_SUFF(pVM);
4484
4485 /*
4486 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4487 */
4488 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4489 {
4490 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4491 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4492 {
4493 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4494 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4495 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4499 }
4500
4501#ifdef VBOX_WITH_REM
4502 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4503 {
4504 Assert(pVM->hm.s.vmx.pRealModeTSS);
4505 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4506 if ( pVCpu->hm.s.vmx.fWasInRealMode
4507 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4508 {
4509 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4510 in real-mode (e.g. OpenBSD 4.0) */
4511 REMFlushTBs(pVM);
4512 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4513 pVCpu->hm.s.vmx.fWasInRealMode = false;
4514 }
4515 }
4516#endif
4517 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4518 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4519 AssertRCReturn(rc, rc);
4520 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4521 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4522 AssertRCReturn(rc, rc);
4523 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4524 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4525 AssertRCReturn(rc, rc);
4526 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4527 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4528 AssertRCReturn(rc, rc);
4529 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4530 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4531 AssertRCReturn(rc, rc);
4532 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4533 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4534 AssertRCReturn(rc, rc);
4535
4536#ifdef VBOX_STRICT
4537 /* Validate. */
4538 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4539#endif
4540
4541 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4542 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4543 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4544 }
4545
4546 /*
4547 * Guest TR.
4548 */
4549 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4550 {
4551 /*
4552 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4553 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4554 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4555 */
4556 uint16_t u16Sel = 0;
4557 uint32_t u32Limit = 0;
4558 uint64_t u64Base = 0;
4559 uint32_t u32AccessRights = 0;
4560
4561 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4562 {
4563 u16Sel = pMixedCtx->tr.Sel;
4564 u32Limit = pMixedCtx->tr.u32Limit;
4565 u64Base = pMixedCtx->tr.u64Base;
4566 u32AccessRights = pMixedCtx->tr.Attr.u;
4567 }
4568 else
4569 {
4570 Assert(pVM->hm.s.vmx.pRealModeTSS);
4571 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4572
4573 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4574 RTGCPHYS GCPhys;
4575 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4576 AssertRCReturn(rc, rc);
4577
4578 X86DESCATTR DescAttr;
4579 DescAttr.u = 0;
4580 DescAttr.n.u1Present = 1;
4581 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4582
4583 u16Sel = 0;
4584 u32Limit = HM_VTX_TSS_SIZE;
4585 u64Base = GCPhys; /* in real-mode phys = virt. */
4586 u32AccessRights = DescAttr.u;
4587 }
4588
4589 /* Validate. */
4590 Assert(!(u16Sel & RT_BIT(2)));
4591 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4592 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4593 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4594 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4595 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4596 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4597 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4598 Assert( (u32Limit & 0xfff) == 0xfff
4599 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4600 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4601 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4602
4603 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4604 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4605 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4606 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4607
4608 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4609 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4610 }
4611
4612 /*
4613 * Guest GDTR.
4614 */
4615 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4616 {
4617 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4618 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4619
4620 /* Validate. */
4621 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4622
4623 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4624 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4625 }
4626
4627 /*
4628 * Guest LDTR.
4629 */
4630 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4631 {
4632 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4633 uint32_t u32Access = 0;
4634 if (!pMixedCtx->ldtr.Attr.u)
4635 u32Access = X86DESCATTR_UNUSABLE;
4636 else
4637 u32Access = pMixedCtx->ldtr.Attr.u;
4638
4639 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4640 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4641 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4642 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4643
4644 /* Validate. */
4645 if (!(u32Access & X86DESCATTR_UNUSABLE))
4646 {
4647 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4648 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4649 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4650 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4651 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4652 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4653 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4654 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4655 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4656 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4657 }
4658
4659 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4660 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4661 }
4662
4663 /*
4664 * Guest IDTR.
4665 */
4666 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4667 {
4668 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4669 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4670
4671 /* Validate. */
4672 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4673
4674 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4675 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4676 }
4677
4678 return VINF_SUCCESS;
4679}
4680
4681
4682/**
4683 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4684 * areas. These MSRs will automatically be loaded to the host CPU on every
4685 * successful VM entry and stored from the host CPU on every successful VM-exit.
4686 *
4687 * This also creates/updates MSR slots for the host MSRs. The actual host
4688 * MSR values are -not- updated here for performance reasons. See
4689 * hmR0VmxSaveHostMsrs().
4690 *
4691 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4692 *
4693 * @returns VBox status code.
4694 * @param pVCpu Pointer to the VMCPU.
4695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4696 * out-of-sync. Make sure to update the required fields
4697 * before using them.
4698 *
4699 * @remarks No-long-jump zone!!!
4700 */
4701static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4702{
4703 AssertPtr(pVCpu);
4704 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4705
4706 /*
4707 * MSRs that we use the auto-load/store MSR area in the VMCS.
4708 */
4709 PVM pVM = pVCpu->CTX_SUFF(pVM);
4710 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4711 {
4712 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4713#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4714 if (pVM->hm.s.fAllow64BitGuests)
4715 {
4716 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4717 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4718 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4719 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4720# ifdef DEBUG
4721 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4722 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4723 {
4724 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4725 pMsr->u64Value));
4726 }
4727# endif
4728 }
4729#endif
4730 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4731 }
4732
4733 /*
4734 * Guest Sysenter MSRs.
4735 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4736 * VM-exits on WRMSRs for these MSRs.
4737 */
4738 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4739 {
4740 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4741 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4742 }
4743
4744 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4745 {
4746 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4747 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4748 }
4749
4750 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4751 {
4752 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4753 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4754 }
4755
4756 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4757 {
4758 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4759 {
4760 /*
4761 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4762 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4763 */
4764 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4765 {
4766 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4767 AssertRCReturn(rc,rc);
4768 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4769 }
4770 else
4771 {
4772 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4773 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4774 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4775 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4776 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4777 }
4778 }
4779 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4780 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4781 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4782 }
4783
4784 return VINF_SUCCESS;
4785}
4786
4787
4788/**
4789 * Loads the guest activity state into the guest-state area in the VMCS.
4790 *
4791 * @returns VBox status code.
4792 * @param pVCpu Pointer to the VMCPU.
4793 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4794 * out-of-sync. Make sure to update the required fields
4795 * before using them.
4796 *
4797 * @remarks No-long-jump zone!!!
4798 */
4799static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4800{
4801 NOREF(pCtx);
4802 /** @todo See if we can make use of other states, e.g.
4803 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4804 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4805 {
4806 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4807 AssertRCReturn(rc, rc);
4808
4809 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4810 }
4811 return VINF_SUCCESS;
4812}
4813
4814
4815/**
4816 * Sets up the appropriate function to run guest code.
4817 *
4818 * @returns VBox status code.
4819 * @param pVCpu Pointer to the VMCPU.
4820 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4821 * out-of-sync. Make sure to update the required fields
4822 * before using them.
4823 *
4824 * @remarks No-long-jump zone!!!
4825 */
4826static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4827{
4828 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4829 {
4830#ifndef VBOX_ENABLE_64_BITS_GUESTS
4831 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4832#endif
4833 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4834#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4835 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4836 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4837 {
4838 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4839 {
4840 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4841 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4842 | HM_CHANGED_VMX_EXIT_CTLS
4843 | HM_CHANGED_VMX_ENTRY_CTLS
4844 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4845 }
4846 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4847 }
4848#else
4849 /* 64-bit host or hybrid host. */
4850 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4851#endif
4852 }
4853 else
4854 {
4855 /* Guest is not in long mode, use the 32-bit handler. */
4856#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4857 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4858 {
4859 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4860 {
4861 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4862 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4863 | HM_CHANGED_VMX_EXIT_CTLS
4864 | HM_CHANGED_VMX_ENTRY_CTLS
4865 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4866 }
4867 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4868 }
4869#else
4870 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4871#endif
4872 }
4873 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4874 return VINF_SUCCESS;
4875}
4876
4877
4878/**
4879 * Wrapper for running the guest code in VT-x.
4880 *
4881 * @returns VBox strict status code.
4882 * @param pVM Pointer to the VM.
4883 * @param pVCpu Pointer to the VMCPU.
4884 * @param pCtx Pointer to the guest-CPU context.
4885 *
4886 * @remarks No-long-jump zone!!!
4887 */
4888DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4889{
4890 /*
4891 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4892 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4893 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4894 */
4895 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4896 /** @todo Add stats for resume vs launch. */
4897#ifdef VBOX_WITH_KERNEL_USING_XMM
4898 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4899#else
4900 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4901#endif
4902}
4903
4904
4905/**
4906 * Reports world-switch error and dumps some useful debug info.
4907 *
4908 * @param pVM Pointer to the VM.
4909 * @param pVCpu Pointer to the VMCPU.
4910 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4911 * @param pCtx Pointer to the guest-CPU context.
4912 * @param pVmxTransient Pointer to the VMX transient structure (only
4913 * exitReason updated).
4914 */
4915static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4916{
4917 Assert(pVM);
4918 Assert(pVCpu);
4919 Assert(pCtx);
4920 Assert(pVmxTransient);
4921 HMVMX_ASSERT_PREEMPT_SAFE();
4922
4923 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4924 switch (rcVMRun)
4925 {
4926 case VERR_VMX_INVALID_VMXON_PTR:
4927 AssertFailed();
4928 break;
4929 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4930 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4931 {
4932 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4933 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4934 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4935 AssertRC(rc);
4936
4937 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4938 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4939 Cannot do it here as we may have been long preempted. */
4940
4941#ifdef VBOX_STRICT
4942 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4943 pVmxTransient->uExitReason));
4944 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4945 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4946 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4947 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4948 else
4949 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4950 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4951 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4952
4953 /* VMX control bits. */
4954 uint32_t u32Val;
4955 uint64_t u64Val;
4956 HMVMXHCUINTREG uHCReg;
4957 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4958 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4959 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4960 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4961 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4962 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4963 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4964 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4965 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4966 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4967 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4968 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4969 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4970 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4971 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4972 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4989 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4990 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4991 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4992 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4993 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4994 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4995 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4996 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4997 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4998 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4999
5000 /* Guest bits. */
5001 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5002 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5003 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5004 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5005 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5006 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5007 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5008 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5009
5010 /* Host bits. */
5011 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5012 Log4(("Host CR0 %#RHr\n", uHCReg));
5013 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5014 Log4(("Host CR3 %#RHr\n", uHCReg));
5015 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5016 Log4(("Host CR4 %#RHr\n", uHCReg));
5017
5018 RTGDTR HostGdtr;
5019 PCX86DESCHC pDesc;
5020 ASMGetGDTR(&HostGdtr);
5021 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5022 Log4(("Host CS %#08x\n", u32Val));
5023 if (u32Val < HostGdtr.cbGdt)
5024 {
5025 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5026 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5027 }
5028
5029 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5030 Log4(("Host DS %#08x\n", u32Val));
5031 if (u32Val < HostGdtr.cbGdt)
5032 {
5033 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5034 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5035 }
5036
5037 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5038 Log4(("Host ES %#08x\n", u32Val));
5039 if (u32Val < HostGdtr.cbGdt)
5040 {
5041 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5042 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5043 }
5044
5045 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5046 Log4(("Host FS %#08x\n", u32Val));
5047 if (u32Val < HostGdtr.cbGdt)
5048 {
5049 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5050 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5051 }
5052
5053 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5054 Log4(("Host GS %#08x\n", u32Val));
5055 if (u32Val < HostGdtr.cbGdt)
5056 {
5057 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5058 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5059 }
5060
5061 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5062 Log4(("Host SS %#08x\n", u32Val));
5063 if (u32Val < HostGdtr.cbGdt)
5064 {
5065 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5066 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5067 }
5068
5069 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5070 Log4(("Host TR %#08x\n", u32Val));
5071 if (u32Val < HostGdtr.cbGdt)
5072 {
5073 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5074 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5075 }
5076
5077 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5078 Log4(("Host TR Base %#RHv\n", uHCReg));
5079 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5080 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5081 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5082 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5083 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5084 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5085 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5086 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5088 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5090 Log4(("Host RSP %#RHv\n", uHCReg));
5091 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5092 Log4(("Host RIP %#RHv\n", uHCReg));
5093# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5094 if (HMVMX_IS_64BIT_HOST_MODE())
5095 {
5096 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5097 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5098 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5099 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5100 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5101 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5102 }
5103# endif
5104#endif /* VBOX_STRICT */
5105 break;
5106 }
5107
5108 default:
5109 /* Impossible */
5110 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5111 break;
5112 }
5113 NOREF(pVM); NOREF(pCtx);
5114}
5115
5116
5117#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5118#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5119# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5120#endif
5121#ifdef VBOX_STRICT
5122static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5123{
5124 switch (idxField)
5125 {
5126 case VMX_VMCS_GUEST_RIP:
5127 case VMX_VMCS_GUEST_RSP:
5128 case VMX_VMCS_GUEST_SYSENTER_EIP:
5129 case VMX_VMCS_GUEST_SYSENTER_ESP:
5130 case VMX_VMCS_GUEST_GDTR_BASE:
5131 case VMX_VMCS_GUEST_IDTR_BASE:
5132 case VMX_VMCS_GUEST_CS_BASE:
5133 case VMX_VMCS_GUEST_DS_BASE:
5134 case VMX_VMCS_GUEST_ES_BASE:
5135 case VMX_VMCS_GUEST_FS_BASE:
5136 case VMX_VMCS_GUEST_GS_BASE:
5137 case VMX_VMCS_GUEST_SS_BASE:
5138 case VMX_VMCS_GUEST_LDTR_BASE:
5139 case VMX_VMCS_GUEST_TR_BASE:
5140 case VMX_VMCS_GUEST_CR3:
5141 return true;
5142 }
5143 return false;
5144}
5145
5146static bool hmR0VmxIsValidReadField(uint32_t idxField)
5147{
5148 switch (idxField)
5149 {
5150 /* Read-only fields. */
5151 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5152 return true;
5153 }
5154 /* Remaining readable fields should also be writable. */
5155 return hmR0VmxIsValidWriteField(idxField);
5156}
5157#endif /* VBOX_STRICT */
5158
5159
5160/**
5161 * Executes the specified handler in 64-bit mode.
5162 *
5163 * @returns VBox status code.
5164 * @param pVM Pointer to the VM.
5165 * @param pVCpu Pointer to the VMCPU.
5166 * @param pCtx Pointer to the guest CPU context.
5167 * @param enmOp The operation to perform.
5168 * @param cbParam Number of parameters.
5169 * @param paParam Array of 32-bit parameters.
5170 */
5171VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5172 uint32_t *paParam)
5173{
5174 int rc, rc2;
5175 PHMGLOBALCPUINFO pCpu;
5176 RTHCPHYS HCPhysCpuPage;
5177 RTCCUINTREG uOldEflags;
5178
5179 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5180 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5181 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5182 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5183
5184#ifdef VBOX_STRICT
5185 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5186 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5187
5188 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5189 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5190#endif
5191
5192 /* Disable interrupts. */
5193 uOldEflags = ASMIntDisableFlags();
5194
5195#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5196 RTCPUID idHostCpu = RTMpCpuId();
5197 CPUMR0SetLApic(pVCpu, idHostCpu);
5198#endif
5199
5200 pCpu = HMR0GetCurrentCpu();
5201 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5202
5203 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5204 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5205
5206 /* Leave VMX Root Mode. */
5207 VMXDisable();
5208
5209 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5210
5211 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5212 CPUMSetHyperEIP(pVCpu, enmOp);
5213 for (int i = (int)cbParam - 1; i >= 0; i--)
5214 CPUMPushHyper(pVCpu, paParam[i]);
5215
5216 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5217
5218 /* Call the switcher. */
5219 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5220 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5221
5222 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5223 /* Make sure the VMX instructions don't cause #UD faults. */
5224 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5225
5226 /* Re-enter VMX Root Mode */
5227 rc2 = VMXEnable(HCPhysCpuPage);
5228 if (RT_FAILURE(rc2))
5229 {
5230 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5231 ASMSetFlags(uOldEflags);
5232 return rc2;
5233 }
5234
5235 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5236 AssertRC(rc2);
5237 Assert(!(ASMGetFlags() & X86_EFL_IF));
5238 ASMSetFlags(uOldEflags);
5239 return rc;
5240}
5241
5242
5243/**
5244 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5245 * supporting 64-bit guests.
5246 *
5247 * @returns VBox status code.
5248 * @param fResume Whether to VMLAUNCH or VMRESUME.
5249 * @param pCtx Pointer to the guest-CPU context.
5250 * @param pCache Pointer to the VMCS cache.
5251 * @param pVM Pointer to the VM.
5252 * @param pVCpu Pointer to the VMCPU.
5253 */
5254DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5255{
5256 uint32_t aParam[6];
5257 PHMGLOBALCPUINFO pCpu = NULL;
5258 RTHCPHYS HCPhysCpuPage = 0;
5259 int rc = VERR_INTERNAL_ERROR_5;
5260
5261 pCpu = HMR0GetCurrentCpu();
5262 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5263
5264#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5265 pCache->uPos = 1;
5266 pCache->interPD = PGMGetInterPaeCR3(pVM);
5267 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5268#endif
5269
5270#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5271 pCache->TestIn.HCPhysCpuPage = 0;
5272 pCache->TestIn.HCPhysVmcs = 0;
5273 pCache->TestIn.pCache = 0;
5274 pCache->TestOut.HCPhysVmcs = 0;
5275 pCache->TestOut.pCache = 0;
5276 pCache->TestOut.pCtx = 0;
5277 pCache->TestOut.eflags = 0;
5278#endif
5279
5280 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5281 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5282 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5283 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5284 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5285 aParam[5] = 0;
5286
5287#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5288 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5289 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5290#endif
5291 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5292
5293#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5294 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5295 Assert(pCtx->dr[4] == 10);
5296 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5297#endif
5298
5299#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5300 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5301 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5302 pVCpu->hm.s.vmx.HCPhysVmcs));
5303 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5304 pCache->TestOut.HCPhysVmcs));
5305 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5306 pCache->TestOut.pCache));
5307 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5308 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5309 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5310 pCache->TestOut.pCtx));
5311 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5312#endif
5313 return rc;
5314}
5315
5316
5317/**
5318 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5319 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5320 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5321 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5322 *
5323 * @returns VBox status code.
5324 * @param pVM Pointer to the VM.
5325 * @param pVCpu Pointer to the VMCPU.
5326 */
5327static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5328{
5329#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5330{ \
5331 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5332 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5333 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5334 ++cReadFields; \
5335}
5336
5337 AssertPtr(pVM);
5338 AssertPtr(pVCpu);
5339 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5340 uint32_t cReadFields = 0;
5341
5342 /*
5343 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5344 * and serve to indicate exceptions to the rules.
5345 */
5346
5347 /* Guest-natural selector base fields. */
5348#if 0
5349 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5352#endif
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5365#if 0
5366 /* Unused natural width guest-state fields. */
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5369#endif
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5372
5373 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5374#if 0
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5384#endif
5385
5386 /* Natural width guest-state fields. */
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5388#if 0
5389 /* Currently unused field. */
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5391#endif
5392
5393 if (pVM->hm.s.fNestedPaging)
5394 {
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5396 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5397 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5398 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5399 }
5400 else
5401 {
5402 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5403 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5404 }
5405
5406#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5407 return VINF_SUCCESS;
5408}
5409
5410
5411/**
5412 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5413 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5414 * darwin, running 64-bit guests).
5415 *
5416 * @returns VBox status code.
5417 * @param pVCpu Pointer to the VMCPU.
5418 * @param idxField The VMCS field encoding.
5419 * @param u64Val 16, 32 or 64-bit value.
5420 */
5421VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5422{
5423 int rc;
5424 switch (idxField)
5425 {
5426 /*
5427 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5428 */
5429 /* 64-bit Control fields. */
5430 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5431 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5432 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5433 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5434 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5435 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5436 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5437 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5438 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5439 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5440 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5441 case VMX_VMCS64_CTRL_EPTP_FULL:
5442 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5443 /* 64-bit Guest-state fields. */
5444 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5445 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5446 case VMX_VMCS64_GUEST_PAT_FULL:
5447 case VMX_VMCS64_GUEST_EFER_FULL:
5448 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5449 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5450 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5451 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5452 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5453 /* 64-bit Host-state fields. */
5454 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5455 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5456 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5457 {
5458 rc = VMXWriteVmcs32(idxField, u64Val);
5459 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5460 break;
5461 }
5462
5463 /*
5464 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5465 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5466 */
5467 /* Natural-width Guest-state fields. */
5468 case VMX_VMCS_GUEST_CR3:
5469 case VMX_VMCS_GUEST_ES_BASE:
5470 case VMX_VMCS_GUEST_CS_BASE:
5471 case VMX_VMCS_GUEST_SS_BASE:
5472 case VMX_VMCS_GUEST_DS_BASE:
5473 case VMX_VMCS_GUEST_FS_BASE:
5474 case VMX_VMCS_GUEST_GS_BASE:
5475 case VMX_VMCS_GUEST_LDTR_BASE:
5476 case VMX_VMCS_GUEST_TR_BASE:
5477 case VMX_VMCS_GUEST_GDTR_BASE:
5478 case VMX_VMCS_GUEST_IDTR_BASE:
5479 case VMX_VMCS_GUEST_RSP:
5480 case VMX_VMCS_GUEST_RIP:
5481 case VMX_VMCS_GUEST_SYSENTER_ESP:
5482 case VMX_VMCS_GUEST_SYSENTER_EIP:
5483 {
5484 if (!(u64Val >> 32))
5485 {
5486 /* If this field is 64-bit, VT-x will zero out the top bits. */
5487 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5488 }
5489 else
5490 {
5491 /* Assert that only the 32->64 switcher case should ever come here. */
5492 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5493 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5494 }
5495 break;
5496 }
5497
5498 default:
5499 {
5500 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5501 rc = VERR_INVALID_PARAMETER;
5502 break;
5503 }
5504 }
5505 AssertRCReturn(rc, rc);
5506 return rc;
5507}
5508
5509
5510/**
5511 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5512 * hosts (except darwin) for 64-bit guests.
5513 *
5514 * @param pVCpu Pointer to the VMCPU.
5515 * @param idxField The VMCS field encoding.
5516 * @param u64Val 16, 32 or 64-bit value.
5517 */
5518VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5519{
5520 AssertPtr(pVCpu);
5521 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5522
5523 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5524 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5525
5526 /* Make sure there are no duplicates. */
5527 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5528 {
5529 if (pCache->Write.aField[i] == idxField)
5530 {
5531 pCache->Write.aFieldVal[i] = u64Val;
5532 return VINF_SUCCESS;
5533 }
5534 }
5535
5536 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5537 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5538 pCache->Write.cValidEntries++;
5539 return VINF_SUCCESS;
5540}
5541
5542/* Enable later when the assembly code uses these as callbacks. */
5543#if 0
5544/*
5545 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5546 *
5547 * @param pVCpu Pointer to the VMCPU.
5548 * @param pCache Pointer to the VMCS cache.
5549 *
5550 * @remarks No-long-jump zone!!!
5551 */
5552VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5553{
5554 AssertPtr(pCache);
5555 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5556 {
5557 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5558 AssertRC(rc);
5559 }
5560 pCache->Write.cValidEntries = 0;
5561}
5562
5563
5564/**
5565 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5566 *
5567 * @param pVCpu Pointer to the VMCPU.
5568 * @param pCache Pointer to the VMCS cache.
5569 *
5570 * @remarks No-long-jump zone!!!
5571 */
5572VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5573{
5574 AssertPtr(pCache);
5575 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5576 {
5577 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5578 AssertRC(rc);
5579 }
5580}
5581#endif
5582#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5583
5584
5585/**
5586 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5587 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5588 * timer.
5589 *
5590 * @returns VBox status code.
5591 * @param pVCpu Pointer to the VMCPU.
5592 *
5593 * @remarks No-long-jump zone!!!
5594 */
5595static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5596{
5597 int rc = VERR_INTERNAL_ERROR_5;
5598 bool fOffsettedTsc = false;
5599 bool fParavirtTsc = false;
5600 PVM pVM = pVCpu->CTX_SUFF(pVM);
5601 if (pVM->hm.s.vmx.fUsePreemptTimer)
5602 {
5603 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5604 &pVCpu->hm.s.vmx.u64TSCOffset);
5605
5606 /* Make sure the returned values have sane upper and lower boundaries. */
5607 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5608 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5609 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5610 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5611
5612 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5613 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5614 }
5615 else
5616 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5617
5618#if 1
5619 if (fParavirtTsc)
5620 {
5621#if 1
5622 uint64_t const u64CurTsc = ASMReadTSC();
5623 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5624 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5625 {
5626 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5627 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5628 }
5629
5630 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5631#endif
5632 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5633 AssertRC(rc);
5634 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5635 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5636
5637 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5638 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5639 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5640 }
5641 else
5642#else
5643 if (fParavirtTsc)
5644 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5645#endif
5646 if (fOffsettedTsc)
5647 {
5648 uint64_t u64CurTSC = ASMReadTSC();
5649 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5650 {
5651 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5652 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5653
5654 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5655 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5656 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5657 }
5658 else
5659 {
5660 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5661 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5662 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5663 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5664 }
5665 }
5666 else
5667 {
5668 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5669 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5670 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5671 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5672 }
5673}
5674
5675
5676/**
5677 * Determines if an exception is a contributory exception. Contributory
5678 * exceptions are ones which can cause double-faults. Page-fault is
5679 * intentionally not included here as it's a conditional contributory exception.
5680 *
5681 * @returns true if the exception is contributory, false otherwise.
5682 * @param uVector The exception vector.
5683 */
5684DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5685{
5686 switch (uVector)
5687 {
5688 case X86_XCPT_GP:
5689 case X86_XCPT_SS:
5690 case X86_XCPT_NP:
5691 case X86_XCPT_TS:
5692 case X86_XCPT_DE:
5693 return true;
5694 default:
5695 break;
5696 }
5697 return false;
5698}
5699
5700
5701/**
5702 * Sets an event as a pending event to be injected into the guest.
5703 *
5704 * @param pVCpu Pointer to the VMCPU.
5705 * @param u32IntInfo The VM-entry interruption-information field.
5706 * @param cbInstr The VM-entry instruction length in bytes (for software
5707 * interrupts, exceptions and privileged software
5708 * exceptions).
5709 * @param u32ErrCode The VM-entry exception error code.
5710 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5711 * page-fault.
5712 *
5713 * @remarks Statistics counter assumes this is a guest event being injected or
5714 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5715 * always incremented.
5716 */
5717DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5718 RTGCUINTPTR GCPtrFaultAddress)
5719{
5720 Assert(!pVCpu->hm.s.Event.fPending);
5721 pVCpu->hm.s.Event.fPending = true;
5722 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5723 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5724 pVCpu->hm.s.Event.cbInstr = cbInstr;
5725 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5726
5727 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5728}
5729
5730
5731/**
5732 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5733 *
5734 * @param pVCpu Pointer to the VMCPU.
5735 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5736 * out-of-sync. Make sure to update the required fields
5737 * before using them.
5738 */
5739DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5740{
5741 NOREF(pMixedCtx);
5742 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5743 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5744 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5745 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5746}
5747
5748
5749/**
5750 * Handle a condition that occurred while delivering an event through the guest
5751 * IDT.
5752 *
5753 * @returns VBox status code (informational error codes included).
5754 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5755 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5756 * continue execution of the guest which will delivery the #DF.
5757 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5758 *
5759 * @param pVCpu Pointer to the VMCPU.
5760 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5761 * out-of-sync. Make sure to update the required fields
5762 * before using them.
5763 * @param pVmxTransient Pointer to the VMX transient structure.
5764 *
5765 * @remarks No-long-jump zone!!!
5766 */
5767static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5768{
5769 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5770
5771 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5772 AssertRCReturn(rc, rc);
5773 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5774 {
5775 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5776 AssertRCReturn(rc, rc);
5777
5778 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5779 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5780
5781 typedef enum
5782 {
5783 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5784 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5785 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5786 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5787 } VMXREFLECTXCPT;
5788
5789 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5790 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5791 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5792 {
5793 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5794 {
5795 enmReflect = VMXREFLECTXCPT_XCPT;
5796#ifdef VBOX_STRICT
5797 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5798 && uExitVector == X86_XCPT_PF)
5799 {
5800 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5801 }
5802#endif
5803 if ( uExitVector == X86_XCPT_PF
5804 && uIdtVector == X86_XCPT_PF)
5805 {
5806 pVmxTransient->fVectoringDoublePF = true;
5807 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5808 }
5809 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5810 && hmR0VmxIsContributoryXcpt(uExitVector)
5811 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5812 || uIdtVector == X86_XCPT_PF))
5813 {
5814 enmReflect = VMXREFLECTXCPT_DF;
5815 }
5816 else if (uIdtVector == X86_XCPT_DF)
5817 enmReflect = VMXREFLECTXCPT_TF;
5818 }
5819 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5820 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5821 {
5822 /*
5823 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5824 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5825 */
5826 enmReflect = VMXREFLECTXCPT_XCPT;
5827
5828 if (uExitVector == X86_XCPT_PF)
5829 {
5830 pVmxTransient->fVectoringPF = true;
5831 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5832 }
5833 }
5834 }
5835 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5836 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5837 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5838 {
5839 /*
5840 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5841 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5842 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5843 */
5844 enmReflect = VMXREFLECTXCPT_XCPT;
5845 }
5846
5847 /*
5848 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5849 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5850 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5851 *
5852 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5853 */
5854 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5855 && enmReflect == VMXREFLECTXCPT_XCPT
5856 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5857 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5858 {
5859 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5860 }
5861
5862 switch (enmReflect)
5863 {
5864 case VMXREFLECTXCPT_XCPT:
5865 {
5866 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5867 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5868 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5869
5870 uint32_t u32ErrCode = 0;
5871 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5872 {
5873 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5874 AssertRCReturn(rc, rc);
5875 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5876 }
5877
5878 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5879 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5880 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5881 rc = VINF_SUCCESS;
5882 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5883 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5884
5885 break;
5886 }
5887
5888 case VMXREFLECTXCPT_DF:
5889 {
5890 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5891 rc = VINF_HM_DOUBLE_FAULT;
5892 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5893 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5894
5895 break;
5896 }
5897
5898 case VMXREFLECTXCPT_TF:
5899 {
5900 rc = VINF_EM_RESET;
5901 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5902 uExitVector));
5903 break;
5904 }
5905
5906 default:
5907 Assert(rc == VINF_SUCCESS);
5908 break;
5909 }
5910 }
5911 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5912 && uExitVector != X86_XCPT_DF
5913 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5914 {
5915 /*
5916 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5917 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5918 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5919 */
5920 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5921 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5922 }
5923
5924 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5925 return rc;
5926}
5927
5928
5929/**
5930 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5931 *
5932 * @returns VBox status code.
5933 * @param pVCpu Pointer to the VMCPU.
5934 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5935 * out-of-sync. Make sure to update the required fields
5936 * before using them.
5937 *
5938 * @remarks No-long-jump zone!!!
5939 */
5940static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5941{
5942 NOREF(pMixedCtx);
5943
5944 /*
5945 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5946 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5947 */
5948 VMMRZCallRing3Disable(pVCpu);
5949 HM_DISABLE_PREEMPT_IF_NEEDED();
5950
5951 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5952 {
5953 uint32_t uVal = 0;
5954 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5955 AssertRCReturn(rc, rc);
5956
5957 uint32_t uShadow = 0;
5958 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5959 AssertRCReturn(rc, rc);
5960
5961 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5962 CPUMSetGuestCR0(pVCpu, uVal);
5963 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5964 }
5965
5966 HM_RESTORE_PREEMPT_IF_NEEDED();
5967 VMMRZCallRing3Enable(pVCpu);
5968 return VINF_SUCCESS;
5969}
5970
5971
5972/**
5973 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5974 *
5975 * @returns VBox status code.
5976 * @param pVCpu Pointer to the VMCPU.
5977 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5978 * out-of-sync. Make sure to update the required fields
5979 * before using them.
5980 *
5981 * @remarks No-long-jump zone!!!
5982 */
5983static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5984{
5985 NOREF(pMixedCtx);
5986
5987 int rc = VINF_SUCCESS;
5988 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5989 {
5990 uint32_t uVal = 0;
5991 uint32_t uShadow = 0;
5992 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5993 AssertRCReturn(rc, rc);
5994 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5995 AssertRCReturn(rc, rc);
5996
5997 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5998 CPUMSetGuestCR4(pVCpu, uVal);
5999 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6000 }
6001 return rc;
6002}
6003
6004
6005/**
6006 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6007 *
6008 * @returns VBox status code.
6009 * @param pVCpu Pointer to the VMCPU.
6010 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6011 * out-of-sync. Make sure to update the required fields
6012 * before using them.
6013 *
6014 * @remarks No-long-jump zone!!!
6015 */
6016static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6017{
6018 int rc = VINF_SUCCESS;
6019 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6020 {
6021 uint64_t u64Val = 0;
6022 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6023 AssertRCReturn(rc, rc);
6024
6025 pMixedCtx->rip = u64Val;
6026 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6027 }
6028 return rc;
6029}
6030
6031
6032/**
6033 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6034 *
6035 * @returns VBox status code.
6036 * @param pVCpu Pointer to the VMCPU.
6037 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6038 * out-of-sync. Make sure to update the required fields
6039 * before using them.
6040 *
6041 * @remarks No-long-jump zone!!!
6042 */
6043static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6044{
6045 int rc = VINF_SUCCESS;
6046 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6047 {
6048 uint64_t u64Val = 0;
6049 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6050 AssertRCReturn(rc, rc);
6051
6052 pMixedCtx->rsp = u64Val;
6053 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6054 }
6055 return rc;
6056}
6057
6058
6059/**
6060 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6061 *
6062 * @returns VBox status code.
6063 * @param pVCpu Pointer to the VMCPU.
6064 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6065 * out-of-sync. Make sure to update the required fields
6066 * before using them.
6067 *
6068 * @remarks No-long-jump zone!!!
6069 */
6070static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6071{
6072 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6073 {
6074 uint32_t uVal = 0;
6075 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6076 AssertRCReturn(rc, rc);
6077
6078 pMixedCtx->eflags.u32 = uVal;
6079 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6080 {
6081 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6082 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6083
6084 pMixedCtx->eflags.Bits.u1VM = 0;
6085 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6086 }
6087
6088 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6089 }
6090 return VINF_SUCCESS;
6091}
6092
6093
6094/**
6095 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6096 * guest-CPU context.
6097 */
6098DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6099{
6100 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6101 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6102 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6103 return rc;
6104}
6105
6106
6107/**
6108 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6109 * from the guest-state area in the VMCS.
6110 *
6111 * @param pVCpu Pointer to the VMCPU.
6112 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6113 * out-of-sync. Make sure to update the required fields
6114 * before using them.
6115 *
6116 * @remarks No-long-jump zone!!!
6117 */
6118static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6119{
6120 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6121 {
6122 uint32_t uIntrState = 0;
6123 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6124 AssertRC(rc);
6125
6126 if (!uIntrState)
6127 {
6128 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6129 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6130
6131 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6133 }
6134 else
6135 {
6136 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6137 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6138 {
6139 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6140 AssertRC(rc);
6141 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6142 AssertRC(rc);
6143
6144 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6145 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6146 }
6147 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6148 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6149
6150 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6151 {
6152 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6153 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6154 }
6155 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6156 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6157 }
6158
6159 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6160 }
6161}
6162
6163
6164/**
6165 * Saves the guest's activity state.
6166 *
6167 * @returns VBox status code.
6168 * @param pVCpu Pointer to the VMCPU.
6169 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6170 * out-of-sync. Make sure to update the required fields
6171 * before using them.
6172 *
6173 * @remarks No-long-jump zone!!!
6174 */
6175static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6176{
6177 NOREF(pMixedCtx);
6178 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6179 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6180 return VINF_SUCCESS;
6181}
6182
6183
6184/**
6185 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6186 * the current VMCS into the guest-CPU context.
6187 *
6188 * @returns VBox status code.
6189 * @param pVCpu Pointer to the VMCPU.
6190 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6191 * out-of-sync. Make sure to update the required fields
6192 * before using them.
6193 *
6194 * @remarks No-long-jump zone!!!
6195 */
6196static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6197{
6198 int rc = VINF_SUCCESS;
6199 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6200 {
6201 uint32_t u32Val = 0;
6202 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6203 pMixedCtx->SysEnter.cs = u32Val;
6204 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6205 }
6206
6207 uint64_t u64Val = 0;
6208 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6209 {
6210 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6211 pMixedCtx->SysEnter.eip = u64Val;
6212 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6213 }
6214 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6215 {
6216 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6217 pMixedCtx->SysEnter.esp = u64Val;
6218 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6219 }
6220 return rc;
6221}
6222
6223
6224/**
6225 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6226 * the CPU back into the guest-CPU context.
6227 *
6228 * @returns VBox status code.
6229 * @param pVCpu Pointer to the VMCPU.
6230 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6231 * out-of-sync. Make sure to update the required fields
6232 * before using them.
6233 *
6234 * @remarks No-long-jump zone!!!
6235 */
6236static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6237{
6238#if HC_ARCH_BITS == 64
6239 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6240 {
6241 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6242 VMMRZCallRing3Disable(pVCpu);
6243 HM_DISABLE_PREEMPT_IF_NEEDED();
6244
6245 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6246 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6247 {
6248 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6249 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6250 }
6251
6252 HM_RESTORE_PREEMPT_IF_NEEDED();
6253 VMMRZCallRing3Enable(pVCpu);
6254 }
6255 else
6256 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6257#else
6258 NOREF(pMixedCtx);
6259 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6260#endif
6261
6262 return VINF_SUCCESS;
6263}
6264
6265
6266/**
6267 * Saves the auto load/store'd guest MSRs from the current VMCS into
6268 * the guest-CPU context.
6269 *
6270 * @returns VBox status code.
6271 * @param pVCpu Pointer to the VMCPU.
6272 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6273 * out-of-sync. Make sure to update the required fields
6274 * before using them.
6275 *
6276 * @remarks No-long-jump zone!!!
6277 */
6278static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6279{
6280 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6281 return VINF_SUCCESS;
6282
6283 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6284 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6285 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6286 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6287 {
6288 switch (pMsr->u32Msr)
6289 {
6290 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6291 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6292 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6293 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6294 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6295 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6296 break;
6297
6298 default:
6299 {
6300 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6301 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6302 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6303 }
6304 }
6305 }
6306
6307 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6308 return VINF_SUCCESS;
6309}
6310
6311
6312/**
6313 * Saves the guest control registers from the current VMCS into the guest-CPU
6314 * context.
6315 *
6316 * @returns VBox status code.
6317 * @param pVCpu Pointer to the VMCPU.
6318 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6319 * out-of-sync. Make sure to update the required fields
6320 * before using them.
6321 *
6322 * @remarks No-long-jump zone!!!
6323 */
6324static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6325{
6326 /* Guest CR0. Guest FPU. */
6327 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6328 AssertRCReturn(rc, rc);
6329
6330 /* Guest CR4. */
6331 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6332 AssertRCReturn(rc, rc);
6333
6334 /* Guest CR2 - updated always during the world-switch or in #PF. */
6335 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6336 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6337 {
6338 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6339 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6340
6341 PVM pVM = pVCpu->CTX_SUFF(pVM);
6342 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6343 || ( pVM->hm.s.fNestedPaging
6344 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6345 {
6346 uint64_t u64Val = 0;
6347 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6348 if (pMixedCtx->cr3 != u64Val)
6349 {
6350 CPUMSetGuestCR3(pVCpu, u64Val);
6351 if (VMMRZCallRing3IsEnabled(pVCpu))
6352 {
6353 PGMUpdateCR3(pVCpu, u64Val);
6354 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6355 }
6356 else
6357 {
6358 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6359 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6360 }
6361 }
6362
6363 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6364 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6365 {
6366 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6367 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6368 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6369 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6370
6371 if (VMMRZCallRing3IsEnabled(pVCpu))
6372 {
6373 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6374 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6375 }
6376 else
6377 {
6378 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6379 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6380 }
6381 }
6382 }
6383
6384 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6385 }
6386
6387 /*
6388 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6389 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6390 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6391 *
6392 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6393 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6394 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6395 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6396 *
6397 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6398 */
6399 if (VMMRZCallRing3IsEnabled(pVCpu))
6400 {
6401 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6402 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6403
6404 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6405 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6406
6407 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6408 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6409 }
6410
6411 return rc;
6412}
6413
6414
6415/**
6416 * Reads a guest segment register from the current VMCS into the guest-CPU
6417 * context.
6418 *
6419 * @returns VBox status code.
6420 * @param pVCpu Pointer to the VMCPU.
6421 * @param idxSel Index of the selector in the VMCS.
6422 * @param idxLimit Index of the segment limit in the VMCS.
6423 * @param idxBase Index of the segment base in the VMCS.
6424 * @param idxAccess Index of the access rights of the segment in the VMCS.
6425 * @param pSelReg Pointer to the segment selector.
6426 *
6427 * @remarks No-long-jump zone!!!
6428 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6429 * macro as that takes care of whether to read from the VMCS cache or
6430 * not.
6431 */
6432DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6433 PCPUMSELREG pSelReg)
6434{
6435 NOREF(pVCpu);
6436
6437 uint32_t u32Val = 0;
6438 int rc = VMXReadVmcs32(idxSel, &u32Val);
6439 AssertRCReturn(rc, rc);
6440 pSelReg->Sel = (uint16_t)u32Val;
6441 pSelReg->ValidSel = (uint16_t)u32Val;
6442 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6443
6444 rc = VMXReadVmcs32(idxLimit, &u32Val);
6445 AssertRCReturn(rc, rc);
6446 pSelReg->u32Limit = u32Val;
6447
6448 uint64_t u64Val = 0;
6449 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6450 AssertRCReturn(rc, rc);
6451 pSelReg->u64Base = u64Val;
6452
6453 rc = VMXReadVmcs32(idxAccess, &u32Val);
6454 AssertRCReturn(rc, rc);
6455 pSelReg->Attr.u = u32Val;
6456
6457 /*
6458 * If VT-x marks the segment as unusable, most other bits remain undefined:
6459 * - For CS the L, D and G bits have meaning.
6460 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6461 * - For the remaining data segments no bits are defined.
6462 *
6463 * The present bit and the unusable bit has been observed to be set at the
6464 * same time (the selector was supposed to be invalid as we started executing
6465 * a V8086 interrupt in ring-0).
6466 *
6467 * What should be important for the rest of the VBox code, is that the P bit is
6468 * cleared. Some of the other VBox code recognizes the unusable bit, but
6469 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6470 * safe side here, we'll strip off P and other bits we don't care about. If
6471 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6472 *
6473 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6474 */
6475 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6476 {
6477 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6478
6479 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6480 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6481 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6482
6483 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6484#ifdef DEBUG_bird
6485 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6486 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6487 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6488#endif
6489 }
6490 return VINF_SUCCESS;
6491}
6492
6493
6494#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6495# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6496 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6497 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6498#else
6499# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6500 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6501 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6502#endif
6503
6504
6505/**
6506 * Saves the guest segment registers from the current VMCS into the guest-CPU
6507 * context.
6508 *
6509 * @returns VBox status code.
6510 * @param pVCpu Pointer to the VMCPU.
6511 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6512 * out-of-sync. Make sure to update the required fields
6513 * before using them.
6514 *
6515 * @remarks No-long-jump zone!!!
6516 */
6517static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6518{
6519 /* Guest segment registers. */
6520 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6521 {
6522 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6523 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6524 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6525 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6526 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6527 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6528 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6529
6530 /* Restore segment attributes for real-on-v86 mode hack. */
6531 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6532 {
6533 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6534 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6535 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6536 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6537 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6538 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6539 }
6540 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6541 }
6542
6543 return VINF_SUCCESS;
6544}
6545
6546
6547/**
6548 * Saves the guest descriptor table registers and task register from the current
6549 * VMCS into the guest-CPU context.
6550 *
6551 * @returns VBox status code.
6552 * @param pVCpu Pointer to the VMCPU.
6553 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6554 * out-of-sync. Make sure to update the required fields
6555 * before using them.
6556 *
6557 * @remarks No-long-jump zone!!!
6558 */
6559static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6560{
6561 int rc = VINF_SUCCESS;
6562
6563 /* Guest LDTR. */
6564 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6565 {
6566 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6567 AssertRCReturn(rc, rc);
6568 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6569 }
6570
6571 /* Guest GDTR. */
6572 uint64_t u64Val = 0;
6573 uint32_t u32Val = 0;
6574 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6575 {
6576 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6577 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6578 pMixedCtx->gdtr.pGdt = u64Val;
6579 pMixedCtx->gdtr.cbGdt = u32Val;
6580 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6581 }
6582
6583 /* Guest IDTR. */
6584 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6585 {
6586 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6587 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6588 pMixedCtx->idtr.pIdt = u64Val;
6589 pMixedCtx->idtr.cbIdt = u32Val;
6590 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6591 }
6592
6593 /* Guest TR. */
6594 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6595 {
6596 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6597 AssertRCReturn(rc, rc);
6598
6599 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6600 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6601 {
6602 rc = VMXLOCAL_READ_SEG(TR, tr);
6603 AssertRCReturn(rc, rc);
6604 }
6605 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6606 }
6607 return rc;
6608}
6609
6610#undef VMXLOCAL_READ_SEG
6611
6612
6613/**
6614 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6615 * context.
6616 *
6617 * @returns VBox status code.
6618 * @param pVCpu Pointer to the VMCPU.
6619 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6620 * out-of-sync. Make sure to update the required fields
6621 * before using them.
6622 *
6623 * @remarks No-long-jump zone!!!
6624 */
6625static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6626{
6627 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6628 {
6629 if (!pVCpu->hm.s.fUsingHyperDR7)
6630 {
6631 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6632 uint32_t u32Val;
6633 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6634 pMixedCtx->dr[7] = u32Val;
6635 }
6636
6637 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6638 }
6639 return VINF_SUCCESS;
6640}
6641
6642
6643/**
6644 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6645 *
6646 * @returns VBox status code.
6647 * @param pVCpu Pointer to the VMCPU.
6648 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6649 * out-of-sync. Make sure to update the required fields
6650 * before using them.
6651 *
6652 * @remarks No-long-jump zone!!!
6653 */
6654static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6655{
6656 NOREF(pMixedCtx);
6657
6658 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6659 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6660 return VINF_SUCCESS;
6661}
6662
6663
6664/**
6665 * Saves the entire guest state from the currently active VMCS into the
6666 * guest-CPU context. This essentially VMREADs all guest-data.
6667 *
6668 * @returns VBox status code.
6669 * @param pVCpu Pointer to the VMCPU.
6670 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6671 * out-of-sync. Make sure to update the required fields
6672 * before using them.
6673 */
6674static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6675{
6676 Assert(pVCpu);
6677 Assert(pMixedCtx);
6678
6679 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6680 return VINF_SUCCESS;
6681
6682 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6683 again on the ring-3 callback path, there is no real need to. */
6684 if (VMMRZCallRing3IsEnabled(pVCpu))
6685 VMMR0LogFlushDisable(pVCpu);
6686 else
6687 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6688 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6689
6690 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6691 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6692
6693 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6694 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6695
6696 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6697 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6698
6699 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6700 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6701
6702 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6703 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6704
6705 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6706 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6707
6708 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6709 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6710
6711 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6712 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6713
6714 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6715 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6716
6717 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6718 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6719
6720 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6721 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6722
6723 if (VMMRZCallRing3IsEnabled(pVCpu))
6724 VMMR0LogFlushEnable(pVCpu);
6725
6726 return rc;
6727}
6728
6729
6730/**
6731 * Check per-VM and per-VCPU force flag actions that require us to go back to
6732 * ring-3 for one reason or another.
6733 *
6734 * @returns VBox status code (information status code included).
6735 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6736 * ring-3.
6737 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6738 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6739 * interrupts)
6740 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6741 * all EMTs to be in ring-3.
6742 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6743 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6744 * to the EM loop.
6745 *
6746 * @param pVM Pointer to the VM.
6747 * @param pVCpu Pointer to the VMCPU.
6748 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6749 * out-of-sync. Make sure to update the required fields
6750 * before using them.
6751 */
6752static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6753{
6754 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6755
6756 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6757 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6758 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6759 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6760 {
6761 /* We need the control registers now, make sure the guest-CPU context is updated. */
6762 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6763 AssertRCReturn(rc3, rc3);
6764
6765 /* Pending HM CR3 sync. */
6766 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6767 {
6768 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6769 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6770 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6771 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6772 }
6773
6774 /* Pending HM PAE PDPEs. */
6775 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6776 {
6777 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6778 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6779 }
6780
6781 /* Pending PGM C3 sync. */
6782 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6783 {
6784 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6785 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6786 if (rc2 != VINF_SUCCESS)
6787 {
6788 AssertRC(rc2);
6789 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6790 return rc2;
6791 }
6792 }
6793
6794 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6795 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6796 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6797 {
6798 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6799 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6800 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6801 return rc2;
6802 }
6803
6804 /* Pending VM request packets, such as hardware interrupts. */
6805 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6806 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6807 {
6808 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6809 return VINF_EM_PENDING_REQUEST;
6810 }
6811
6812 /* Pending PGM pool flushes. */
6813 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6814 {
6815 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6816 return VINF_PGM_POOL_FLUSH_PENDING;
6817 }
6818
6819 /* Pending DMA requests. */
6820 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6821 {
6822 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6823 return VINF_EM_RAW_TO_R3;
6824 }
6825 }
6826
6827 return VINF_SUCCESS;
6828}
6829
6830
6831/**
6832 * Converts any TRPM trap into a pending HM event. This is typically used when
6833 * entering from ring-3 (not longjmp returns).
6834 *
6835 * @param pVCpu Pointer to the VMCPU.
6836 */
6837static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6838{
6839 Assert(TRPMHasTrap(pVCpu));
6840 Assert(!pVCpu->hm.s.Event.fPending);
6841
6842 uint8_t uVector;
6843 TRPMEVENT enmTrpmEvent;
6844 RTGCUINT uErrCode;
6845 RTGCUINTPTR GCPtrFaultAddress;
6846 uint8_t cbInstr;
6847
6848 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6849 AssertRC(rc);
6850
6851 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6852 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6853 if (enmTrpmEvent == TRPM_TRAP)
6854 {
6855 switch (uVector)
6856 {
6857 case X86_XCPT_NMI:
6858 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6859 break;
6860
6861 case X86_XCPT_BP:
6862 case X86_XCPT_OF:
6863 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6864 break;
6865
6866 case X86_XCPT_PF:
6867 case X86_XCPT_DF:
6868 case X86_XCPT_TS:
6869 case X86_XCPT_NP:
6870 case X86_XCPT_SS:
6871 case X86_XCPT_GP:
6872 case X86_XCPT_AC:
6873 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6874 /* no break! */
6875 default:
6876 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6877 break;
6878 }
6879 }
6880 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6881 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6882 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6883 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6884 else
6885 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6886
6887 rc = TRPMResetTrap(pVCpu);
6888 AssertRC(rc);
6889 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6890 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6891
6892 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6893 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6894}
6895
6896
6897/**
6898 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6899 * VT-x to execute any instruction.
6900 *
6901 * @param pvCpu Pointer to the VMCPU.
6902 */
6903static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6904{
6905 Assert(pVCpu->hm.s.Event.fPending);
6906
6907 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6908 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6909 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6910 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6911
6912 /* If a trap was already pending, we did something wrong! */
6913 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6914
6915 TRPMEVENT enmTrapType;
6916 switch (uVectorType)
6917 {
6918 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6919 enmTrapType = TRPM_HARDWARE_INT;
6920 break;
6921
6922 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6923 enmTrapType = TRPM_SOFTWARE_INT;
6924 break;
6925
6926 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6927 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6928 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6929 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6930 enmTrapType = TRPM_TRAP;
6931 break;
6932
6933 default:
6934 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6935 enmTrapType = TRPM_32BIT_HACK;
6936 break;
6937 }
6938
6939 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6940
6941 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6942 AssertRC(rc);
6943
6944 if (fErrorCodeValid)
6945 TRPMSetErrorCode(pVCpu, uErrorCode);
6946
6947 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6948 && uVector == X86_XCPT_PF)
6949 {
6950 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6951 }
6952 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6953 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6954 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6955 {
6956 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6957 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6958 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6959 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6960 }
6961 pVCpu->hm.s.Event.fPending = false;
6962}
6963
6964
6965/**
6966 * Does the necessary state syncing before returning to ring-3 for any reason
6967 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6968 *
6969 * @returns VBox status code.
6970 * @param pVM Pointer to the VM.
6971 * @param pVCpu Pointer to the VMCPU.
6972 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6973 * be out-of-sync. Make sure to update the required
6974 * fields before using them.
6975 * @param fSaveGuestState Whether to save the guest state or not.
6976 *
6977 * @remarks No-long-jmp zone!!!
6978 */
6979static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6980{
6981 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6982 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6983
6984 RTCPUID idCpu = RTMpCpuId();
6985 Log4Func(("HostCpuId=%u\n", idCpu));
6986
6987 /*
6988 * !!! IMPORTANT !!!
6989 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6990 */
6991
6992 /* Save the guest state if necessary. */
6993 if ( fSaveGuestState
6994 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6995 {
6996 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6997 AssertRCReturn(rc, rc);
6998 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6999 }
7000
7001 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7002 if (CPUMIsGuestFPUStateActive(pVCpu))
7003 {
7004 /* We shouldn't reload CR0 without saving it first. */
7005 if (!fSaveGuestState)
7006 {
7007 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7008 AssertRCReturn(rc, rc);
7009 }
7010 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7011 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7012 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7013 }
7014
7015 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7016#ifdef VBOX_STRICT
7017 if (CPUMIsHyperDebugStateActive(pVCpu))
7018 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7019#endif
7020 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7021 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7022 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7023 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7024
7025#if HC_ARCH_BITS == 64
7026 /* Restore host-state bits that VT-x only restores partially. */
7027 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7028 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7029 {
7030 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7031 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7032 }
7033 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7034#endif
7035
7036#if HC_ARCH_BITS == 64
7037 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7038 if ( pVM->hm.s.fAllow64BitGuests
7039 && pVCpu->hm.s.vmx.fLazyMsrs)
7040 {
7041 /* We shouldn't reload the guest MSRs without saving it first. */
7042 if (!fSaveGuestState)
7043 {
7044 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7045 AssertRCReturn(rc, rc);
7046 }
7047 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7048 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7049 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7050 }
7051#endif
7052
7053 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7054 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7055
7056 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7057 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7058 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7059 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7060 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7061 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7062 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7063 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7064
7065 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7066
7067 /** @todo This partially defeats the purpose of having preemption hooks.
7068 * The problem is, deregistering the hooks should be moved to a place that
7069 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7070 * context.
7071 */
7072 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7073 {
7074 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7075 AssertRCReturn(rc, rc);
7076
7077 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7078 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7079 }
7080 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7081 NOREF(idCpu);
7082
7083 return VINF_SUCCESS;
7084}
7085
7086
7087/**
7088 * Leaves the VT-x session.
7089 *
7090 * @returns VBox status code.
7091 * @param pVM Pointer to the VM.
7092 * @param pVCpu Pointer to the VMCPU.
7093 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7094 * out-of-sync. Make sure to update the required fields
7095 * before using them.
7096 *
7097 * @remarks No-long-jmp zone!!!
7098 */
7099DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7100{
7101 HM_DISABLE_PREEMPT_IF_NEEDED();
7102 HMVMX_ASSERT_CPU_SAFE();
7103 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7104 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7105
7106 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7107 and done this from the VMXR0ThreadCtxCallback(). */
7108 if (!pVCpu->hm.s.fLeaveDone)
7109 {
7110 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7111 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7112 pVCpu->hm.s.fLeaveDone = true;
7113 }
7114 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7115
7116 /*
7117 * !!! IMPORTANT !!!
7118 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7119 */
7120
7121 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7122 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7123 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7124 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7125 VMMR0ThreadCtxHooksDeregister(pVCpu);
7126
7127 /* Leave HM context. This takes care of local init (term). */
7128 int rc = HMR0LeaveCpu(pVCpu);
7129
7130 HM_RESTORE_PREEMPT_IF_NEEDED();
7131
7132 return rc;
7133}
7134
7135
7136/**
7137 * Does the necessary state syncing before doing a longjmp to ring-3.
7138 *
7139 * @returns VBox status code.
7140 * @param pVM Pointer to the VM.
7141 * @param pVCpu Pointer to the VMCPU.
7142 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7143 * out-of-sync. Make sure to update the required fields
7144 * before using them.
7145 *
7146 * @remarks No-long-jmp zone!!!
7147 */
7148DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7149{
7150 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7151}
7152
7153
7154/**
7155 * Take necessary actions before going back to ring-3.
7156 *
7157 * An action requires us to go back to ring-3. This function does the necessary
7158 * steps before we can safely return to ring-3. This is not the same as longjmps
7159 * to ring-3, this is voluntary and prepares the guest so it may continue
7160 * executing outside HM (recompiler/IEM).
7161 *
7162 * @returns VBox status code.
7163 * @param pVM Pointer to the VM.
7164 * @param pVCpu Pointer to the VMCPU.
7165 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7166 * out-of-sync. Make sure to update the required fields
7167 * before using them.
7168 * @param rcExit The reason for exiting to ring-3. Can be
7169 * VINF_VMM_UNKNOWN_RING3_CALL.
7170 */
7171static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7172{
7173 Assert(pVM);
7174 Assert(pVCpu);
7175 Assert(pMixedCtx);
7176 HMVMX_ASSERT_PREEMPT_SAFE();
7177
7178 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7179 {
7180 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7181 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7182 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7183 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7184 }
7185
7186 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7187 VMMRZCallRing3Disable(pVCpu);
7188 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7189
7190 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7191 if (pVCpu->hm.s.Event.fPending)
7192 {
7193 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7194 Assert(!pVCpu->hm.s.Event.fPending);
7195 }
7196
7197 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7198 and if we're injecting an event we should have a TRPM trap pending. */
7199 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7200 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7201
7202 /* Save guest state and restore host state bits. */
7203 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7204 AssertRCReturn(rc, rc);
7205 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7206 /* Thread-context hooks are unregistered at this point!!! */
7207
7208 /* Sync recompiler state. */
7209 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7210 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7211 | CPUM_CHANGED_LDTR
7212 | CPUM_CHANGED_GDTR
7213 | CPUM_CHANGED_IDTR
7214 | CPUM_CHANGED_TR
7215 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7216 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7217 if ( pVM->hm.s.fNestedPaging
7218 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7219 {
7220 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7221 }
7222
7223 Assert(!pVCpu->hm.s.fClearTrapFlag);
7224
7225 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7226 if (rcExit != VINF_EM_RAW_INTERRUPT)
7227 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7228
7229 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7230
7231 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7232 VMMRZCallRing3RemoveNotification(pVCpu);
7233 VMMRZCallRing3Enable(pVCpu);
7234
7235 return rc;
7236}
7237
7238
7239/**
7240 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7241 * longjump to ring-3 and possibly get preempted.
7242 *
7243 * @returns VBox status code.
7244 * @param pVCpu Pointer to the VMCPU.
7245 * @param enmOperation The operation causing the ring-3 longjump.
7246 * @param pvUser Opaque pointer to the guest-CPU context. The data
7247 * may be out-of-sync. Make sure to update the required
7248 * fields before using them.
7249 */
7250DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7251{
7252 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7253 {
7254 /*
7255 * !!! IMPORTANT !!!
7256 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7257 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7258 */
7259 VMMRZCallRing3RemoveNotification(pVCpu);
7260 VMMRZCallRing3Disable(pVCpu);
7261 HM_DISABLE_PREEMPT_IF_NEEDED();
7262
7263 PVM pVM = pVCpu->CTX_SUFF(pVM);
7264 if (CPUMIsGuestFPUStateActive(pVCpu))
7265 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7266
7267 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7268
7269#if HC_ARCH_BITS == 64
7270 /* Restore host-state bits that VT-x only restores partially. */
7271 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7272 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7273 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7274 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7275
7276 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7277 if ( pVM->hm.s.fAllow64BitGuests
7278 && pVCpu->hm.s.vmx.fLazyMsrs)
7279 {
7280 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7281 }
7282#endif
7283 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7284 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7285 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7286 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7287 {
7288 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7289 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7290 }
7291
7292 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7293 VMMR0ThreadCtxHooksDeregister(pVCpu);
7294
7295 HMR0LeaveCpu(pVCpu);
7296 HM_RESTORE_PREEMPT_IF_NEEDED();
7297 return VINF_SUCCESS;
7298 }
7299
7300 Assert(pVCpu);
7301 Assert(pvUser);
7302 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7303 HMVMX_ASSERT_PREEMPT_SAFE();
7304
7305 VMMRZCallRing3Disable(pVCpu);
7306 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7307
7308 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7309 enmOperation));
7310
7311 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7312 AssertRCReturn(rc, rc);
7313
7314 VMMRZCallRing3Enable(pVCpu);
7315 return VINF_SUCCESS;
7316}
7317
7318
7319/**
7320 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7321 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7322 *
7323 * @param pVCpu Pointer to the VMCPU.
7324 */
7325DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7326{
7327 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7328 {
7329 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7330 {
7331 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7332 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7333 AssertRC(rc);
7334 Log4(("Setup interrupt-window exiting\n"));
7335 }
7336 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7337}
7338
7339
7340/**
7341 * Clears the interrupt-window exiting control in the VMCS.
7342 *
7343 * @param pVCpu Pointer to the VMCPU.
7344 */
7345DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7346{
7347 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7348 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7349 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7350 AssertRC(rc);
7351 Log4(("Cleared interrupt-window exiting\n"));
7352}
7353
7354
7355/**
7356 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7357 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7358 *
7359 * @param pVCpu Pointer to the VMCPU.
7360 */
7361DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7362{
7363 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7364 {
7365 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7366 {
7367 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7368 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7369 AssertRC(rc);
7370 Log4(("Setup NMI-window exiting\n"));
7371 }
7372 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7373}
7374
7375
7376/**
7377 * Clears the NMI-window exiting control in the VMCS.
7378 *
7379 * @param pVCpu Pointer to the VMCPU.
7380 */
7381DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7382{
7383 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7384 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7385 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7386 AssertRC(rc);
7387 Log4(("Cleared NMI-window exiting\n"));
7388}
7389
7390
7391/**
7392 * Evaluates the event to be delivered to the guest and sets it as the pending
7393 * event.
7394 *
7395 * @param pVCpu Pointer to the VMCPU.
7396 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7397 * out-of-sync. Make sure to update the required fields
7398 * before using them.
7399 */
7400static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7401{
7402 Assert(!pVCpu->hm.s.Event.fPending);
7403
7404 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7405 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7406 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7407 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7408 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7409
7410 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7411 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7412 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7413 Assert(!TRPMHasTrap(pVCpu));
7414
7415 /*
7416 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7417 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7418 */
7419 /** @todo SMI. SMIs take priority over NMIs. */
7420 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7421 {
7422 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7423 if ( !fBlockNmi
7424 && !fBlockSti
7425 && !fBlockMovSS)
7426 {
7427 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7428 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7429 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7430
7431 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7432 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7433 }
7434 else
7435 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7436 }
7437 /*
7438 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7439 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7440 */
7441 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7442 && !pVCpu->hm.s.fSingleInstruction)
7443 {
7444 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7445 AssertRC(rc);
7446 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7447 if ( !fBlockInt
7448 && !fBlockSti
7449 && !fBlockMovSS)
7450 {
7451 uint8_t u8Interrupt;
7452 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7453 if (RT_SUCCESS(rc))
7454 {
7455 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7456 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7457 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7458
7459 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7460 }
7461 else
7462 {
7463 /** @todo Does this actually happen? If not turn it into an assertion. */
7464 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7465 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7466 }
7467 }
7468 else
7469 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7470 }
7471}
7472
7473
7474/**
7475 * Sets a pending-debug exception to be delivered to the guest if the guest is
7476 * single-stepping.
7477 *
7478 * @param pVCpu Pointer to the VMCPU.
7479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7480 * out-of-sync. Make sure to update the required fields
7481 * before using them.
7482 */
7483DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7484{
7485 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7486 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7487 {
7488 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7489 AssertRC(rc);
7490 }
7491}
7492
7493
7494/**
7495 * Injects any pending events into the guest if the guest is in a state to
7496 * receive them.
7497 *
7498 * @returns VBox status code (informational status codes included).
7499 * @param pVCpu Pointer to the VMCPU.
7500 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7501 * out-of-sync. Make sure to update the required fields
7502 * before using them.
7503 */
7504static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7505{
7506 HMVMX_ASSERT_PREEMPT_SAFE();
7507 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7508
7509 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7510 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7511 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7512 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7513
7514 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7515 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7516 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7517 Assert(!TRPMHasTrap(pVCpu));
7518
7519 int rc = VINF_SUCCESS;
7520 if (pVCpu->hm.s.Event.fPending)
7521 {
7522 /*
7523 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7524 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7525 * ended up enabling interrupts outside VT-x.
7526 */
7527 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7528 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7529 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7530 {
7531 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7532 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7533 }
7534
7535#ifdef VBOX_STRICT
7536 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7537 {
7538 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7539 Assert(!fBlockInt);
7540 Assert(!fBlockSti);
7541 Assert(!fBlockMovSS);
7542 }
7543 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7544 {
7545 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7546 Assert(!fBlockSti);
7547 Assert(!fBlockMovSS);
7548 Assert(!fBlockNmi);
7549 }
7550#endif
7551 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7552 (uint8_t)uIntType));
7553 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7554 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7555 AssertRCReturn(rc, rc);
7556
7557 /* Update the interruptibility-state as it could have been changed by
7558 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7559 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7560 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7561
7562#ifdef VBOX_WITH_STATISTICS
7563 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7564 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7565 else
7566 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7567#endif
7568 }
7569
7570 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7571 if ( fBlockSti
7572 || fBlockMovSS)
7573 {
7574 if ( !pVCpu->hm.s.fSingleInstruction
7575 && !DBGFIsStepping(pVCpu))
7576 {
7577 /*
7578 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7579 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7580 * See Intel spec. 27.3.4 "Saving Non-Register State".
7581 */
7582 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7583 AssertRCReturn(rc2, rc2);
7584 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7585 }
7586 else if (pMixedCtx->eflags.Bits.u1TF)
7587 {
7588 /*
7589 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7590 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7591 */
7592 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7593 uIntrState = 0;
7594 }
7595 }
7596
7597 /*
7598 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7599 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7600 */
7601 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7602 AssertRC(rc2);
7603
7604 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7605 NOREF(fBlockMovSS); NOREF(fBlockSti);
7606 return rc;
7607}
7608
7609
7610/**
7611 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7612 *
7613 * @param pVCpu Pointer to the VMCPU.
7614 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7615 * out-of-sync. Make sure to update the required fields
7616 * before using them.
7617 */
7618DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7619{
7620 NOREF(pMixedCtx);
7621 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7622 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7623}
7624
7625
7626/**
7627 * Injects a double-fault (#DF) exception into the VM.
7628 *
7629 * @returns VBox status code (informational status code included).
7630 * @param pVCpu Pointer to the VMCPU.
7631 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7632 * out-of-sync. Make sure to update the required fields
7633 * before using them.
7634 */
7635DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7636{
7637 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7638 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7639 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7640 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7641 puIntrState);
7642}
7643
7644
7645/**
7646 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7647 *
7648 * @param pVCpu Pointer to the VMCPU.
7649 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7650 * out-of-sync. Make sure to update the required fields
7651 * before using them.
7652 */
7653DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7654{
7655 NOREF(pMixedCtx);
7656 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7657 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7658 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7659}
7660
7661
7662/**
7663 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7664 *
7665 * @param pVCpu Pointer to the VMCPU.
7666 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7667 * out-of-sync. Make sure to update the required fields
7668 * before using them.
7669 * @param cbInstr The value of RIP that is to be pushed on the guest
7670 * stack.
7671 */
7672DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7673{
7674 NOREF(pMixedCtx);
7675 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7676 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7677 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7678}
7679
7680
7681/**
7682 * Injects a general-protection (#GP) fault into the VM.
7683 *
7684 * @returns VBox status code (informational status code included).
7685 * @param pVCpu Pointer to the VMCPU.
7686 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7687 * out-of-sync. Make sure to update the required fields
7688 * before using them.
7689 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7690 * mode, i.e. in real-mode it's not valid).
7691 * @param u32ErrorCode The error code associated with the #GP.
7692 */
7693DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7694 uint32_t *puIntrState)
7695{
7696 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7697 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7698 if (fErrorCodeValid)
7699 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7700 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7701 puIntrState);
7702}
7703
7704
7705/**
7706 * Sets a general-protection (#GP) exception as pending-for-injection into the
7707 * VM.
7708 *
7709 * @param pVCpu Pointer to the VMCPU.
7710 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7711 * out-of-sync. Make sure to update the required fields
7712 * before using them.
7713 * @param u32ErrorCode The error code associated with the #GP.
7714 */
7715DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7716{
7717 NOREF(pMixedCtx);
7718 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7719 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7720 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7721 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7722}
7723
7724
7725/**
7726 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7727 *
7728 * @param pVCpu Pointer to the VMCPU.
7729 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7730 * out-of-sync. Make sure to update the required fields
7731 * before using them.
7732 * @param uVector The software interrupt vector number.
7733 * @param cbInstr The value of RIP that is to be pushed on the guest
7734 * stack.
7735 */
7736DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7737{
7738 NOREF(pMixedCtx);
7739 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7740 if ( uVector == X86_XCPT_BP
7741 || uVector == X86_XCPT_OF)
7742 {
7743 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7744 }
7745 else
7746 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7747 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7748}
7749
7750
7751/**
7752 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7753 * stack.
7754 *
7755 * @returns VBox status code (information status code included).
7756 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7757 * @param pVM Pointer to the VM.
7758 * @param pMixedCtx Pointer to the guest-CPU context.
7759 * @param uValue The value to push to the guest stack.
7760 */
7761DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7762{
7763 /*
7764 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7765 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7766 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7767 */
7768 if (pMixedCtx->sp == 1)
7769 return VINF_EM_RESET;
7770 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7771 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7772 AssertRCReturn(rc, rc);
7773 return rc;
7774}
7775
7776
7777/**
7778 * Injects an event into the guest upon VM-entry by updating the relevant fields
7779 * in the VM-entry area in the VMCS.
7780 *
7781 * @returns VBox status code (informational error codes included).
7782 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7783 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7784 *
7785 * @param pVCpu Pointer to the VMCPU.
7786 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7787 * be out-of-sync. Make sure to update the required
7788 * fields before using them.
7789 * @param u64IntInfo The VM-entry interruption-information field.
7790 * @param cbInstr The VM-entry instruction length in bytes (for
7791 * software interrupts, exceptions and privileged
7792 * software exceptions).
7793 * @param u32ErrCode The VM-entry exception error code.
7794 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7795 * @param puIntrState Pointer to the current guest interruptibility-state.
7796 * This interruptibility-state will be updated if
7797 * necessary. This cannot not be NULL.
7798 *
7799 * @remarks Requires CR0!
7800 * @remarks No-long-jump zone!!!
7801 */
7802static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7803 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7804{
7805 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7806 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7807 Assert(puIntrState);
7808 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7809
7810 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7811 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7812
7813#ifdef VBOX_STRICT
7814 /* Validate the error-code-valid bit for hardware exceptions. */
7815 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7816 {
7817 switch (uVector)
7818 {
7819 case X86_XCPT_PF:
7820 case X86_XCPT_DF:
7821 case X86_XCPT_TS:
7822 case X86_XCPT_NP:
7823 case X86_XCPT_SS:
7824 case X86_XCPT_GP:
7825 case X86_XCPT_AC:
7826 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7827 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7828 /* fallthru */
7829 default:
7830 break;
7831 }
7832 }
7833#endif
7834
7835 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7836 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7837 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7838
7839 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7840
7841 /* We require CR0 to check if the guest is in real-mode. */
7842 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7843 AssertRCReturn(rc, rc);
7844
7845 /*
7846 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7847 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7848 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7849 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7850 */
7851 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7852 {
7853 PVM pVM = pVCpu->CTX_SUFF(pVM);
7854 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7855 {
7856 Assert(PDMVmmDevHeapIsEnabled(pVM));
7857 Assert(pVM->hm.s.vmx.pRealModeTSS);
7858
7859 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7860 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7861 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7862 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7863 AssertRCReturn(rc, rc);
7864 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7865
7866 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7867 size_t const cbIdtEntry = sizeof(X86IDTR16);
7868 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7869 {
7870 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7871 if (uVector == X86_XCPT_DF)
7872 return VINF_EM_RESET;
7873 else if (uVector == X86_XCPT_GP)
7874 {
7875 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7876 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7877 }
7878
7879 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7880 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7881 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7882 }
7883
7884 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7885 uint16_t uGuestIp = pMixedCtx->ip;
7886 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7887 {
7888 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7889 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7890 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7891 }
7892 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7893 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7894
7895 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7896 X86IDTR16 IdtEntry;
7897 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7898 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7899 AssertRCReturn(rc, rc);
7900
7901 /* Construct the stack frame for the interrupt/exception handler. */
7902 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7903 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7904 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7905 AssertRCReturn(rc, rc);
7906
7907 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7908 if (rc == VINF_SUCCESS)
7909 {
7910 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7911 pMixedCtx->rip = IdtEntry.offSel;
7912 pMixedCtx->cs.Sel = IdtEntry.uSel;
7913 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7914 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7915 && uVector == X86_XCPT_PF)
7916 {
7917 pMixedCtx->cr2 = GCPtrFaultAddress;
7918 }
7919
7920 /* If any other guest-state bits are changed here, make sure to update
7921 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7922 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7923 | HM_CHANGED_GUEST_RIP
7924 | HM_CHANGED_GUEST_RFLAGS
7925 | HM_CHANGED_GUEST_RSP);
7926
7927 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7928 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7929 {
7930 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7931 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7932 Log4(("Clearing inhibition due to STI.\n"));
7933 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7934 }
7935 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7936
7937 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7938 it, if we are returning to ring-3 before executing guest code. */
7939 pVCpu->hm.s.Event.fPending = false;
7940 }
7941 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7942 return rc;
7943 }
7944 else
7945 {
7946 /*
7947 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7948 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7949 */
7950 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7951 }
7952 }
7953
7954 /* Validate. */
7955 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7956 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7957 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7958
7959 /* Inject. */
7960 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7961 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7962 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7963 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7964
7965 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7966 && uVector == X86_XCPT_PF)
7967 {
7968 pMixedCtx->cr2 = GCPtrFaultAddress;
7969 }
7970
7971 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7972 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7973
7974 AssertRCReturn(rc, rc);
7975 return rc;
7976}
7977
7978
7979/**
7980 * Clears the interrupt-window exiting control in the VMCS and if necessary
7981 * clears the current event in the VMCS as well.
7982 *
7983 * @returns VBox status code.
7984 * @param pVCpu Pointer to the VMCPU.
7985 *
7986 * @remarks Use this function only to clear events that have not yet been
7987 * delivered to the guest but are injected in the VMCS!
7988 * @remarks No-long-jump zone!!!
7989 */
7990static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7991{
7992 int rc;
7993 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7994
7995 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7996 {
7997 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7998 Assert(!pVCpu->hm.s.Event.fPending);
7999 }
8000
8001 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8002 {
8003 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8004 Assert(!pVCpu->hm.s.Event.fPending);
8005 }
8006
8007 if (!pVCpu->hm.s.Event.fPending)
8008 return;
8009
8010#ifdef VBOX_STRICT
8011 uint32_t u32EntryInfo;
8012 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8013 AssertRC(rc);
8014 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8015#endif
8016
8017 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8018 AssertRC(rc);
8019
8020 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8021 AssertRC(rc);
8022
8023 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8024 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8025}
8026
8027
8028/**
8029 * Enters the VT-x session.
8030 *
8031 * @returns VBox status code.
8032 * @param pVM Pointer to the VM.
8033 * @param pVCpu Pointer to the VMCPU.
8034 * @param pCpu Pointer to the CPU info struct.
8035 */
8036VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8037{
8038 AssertPtr(pVM);
8039 AssertPtr(pVCpu);
8040 Assert(pVM->hm.s.vmx.fSupported);
8041 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8042 NOREF(pCpu); NOREF(pVM);
8043
8044 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8045 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8046
8047#ifdef VBOX_STRICT
8048 /* Make sure we're in VMX root mode. */
8049 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8050 if (!(u32HostCR4 & X86_CR4_VMXE))
8051 {
8052 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8053 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8054 }
8055#endif
8056
8057 /*
8058 * Load the VCPU's VMCS as the current (and active) one.
8059 */
8060 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8061 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8062 if (RT_FAILURE(rc))
8063 return rc;
8064
8065 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8066 pVCpu->hm.s.fLeaveDone = false;
8067 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8068
8069 return VINF_SUCCESS;
8070}
8071
8072
8073/**
8074 * The thread-context callback (only on platforms which support it).
8075 *
8076 * @param enmEvent The thread-context event.
8077 * @param pVCpu Pointer to the VMCPU.
8078 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8079 * @thread EMT(pVCpu)
8080 */
8081VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8082{
8083 NOREF(fGlobalInit);
8084
8085 switch (enmEvent)
8086 {
8087 case RTTHREADCTXEVENT_PREEMPTING:
8088 {
8089 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8090 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8091 VMCPU_ASSERT_EMT(pVCpu);
8092
8093 PVM pVM = pVCpu->CTX_SUFF(pVM);
8094 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8095
8096 /* No longjmps (logger flushes, locks) in this fragile context. */
8097 VMMRZCallRing3Disable(pVCpu);
8098 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8099
8100 /*
8101 * Restore host-state (FPU, debug etc.)
8102 */
8103 if (!pVCpu->hm.s.fLeaveDone)
8104 {
8105 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8106 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8107 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8108 pVCpu->hm.s.fLeaveDone = true;
8109 }
8110
8111 /* Leave HM context, takes care of local init (term). */
8112 int rc = HMR0LeaveCpu(pVCpu);
8113 AssertRC(rc); NOREF(rc);
8114
8115 /* Restore longjmp state. */
8116 VMMRZCallRing3Enable(pVCpu);
8117 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8118 break;
8119 }
8120
8121 case RTTHREADCTXEVENT_RESUMED:
8122 {
8123 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8124 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8125 VMCPU_ASSERT_EMT(pVCpu);
8126
8127 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8128 VMMRZCallRing3Disable(pVCpu);
8129 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8130
8131 /* Initialize the bare minimum state required for HM. This takes care of
8132 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8133 int rc = HMR0EnterCpu(pVCpu);
8134 AssertRC(rc);
8135 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8136
8137 /* Load the active VMCS as the current one. */
8138 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8139 {
8140 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8141 AssertRC(rc); NOREF(rc);
8142 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8143 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8144 }
8145 pVCpu->hm.s.fLeaveDone = false;
8146
8147 /* Restore longjmp state. */
8148 VMMRZCallRing3Enable(pVCpu);
8149 break;
8150 }
8151
8152 default:
8153 break;
8154 }
8155}
8156
8157
8158/**
8159 * Saves the host state in the VMCS host-state.
8160 * Sets up the VM-exit MSR-load area.
8161 *
8162 * The CPU state will be loaded from these fields on every successful VM-exit.
8163 *
8164 * @returns VBox status code.
8165 * @param pVM Pointer to the VM.
8166 * @param pVCpu Pointer to the VMCPU.
8167 *
8168 * @remarks No-long-jump zone!!!
8169 */
8170static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8171{
8172 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8173
8174 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8175 return VINF_SUCCESS;
8176
8177 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8178 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8179
8180 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8181 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8182
8183 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8184 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8185
8186 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8187 return rc;
8188}
8189
8190
8191/**
8192 * Saves the host state in the VMCS host-state.
8193 *
8194 * @returns VBox status code.
8195 * @param pVM Pointer to the VM.
8196 * @param pVCpu Pointer to the VMCPU.
8197 *
8198 * @remarks No-long-jump zone!!!
8199 */
8200VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8201{
8202 AssertPtr(pVM);
8203 AssertPtr(pVCpu);
8204
8205 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8206
8207 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8208 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8209 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8210 return hmR0VmxSaveHostState(pVM, pVCpu);
8211}
8212
8213
8214/**
8215 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8216 * loaded from these fields on every successful VM-entry.
8217 *
8218 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8219 * Sets up the VM-entry controls.
8220 * Sets up the appropriate VMX non-root function to execute guest code based on
8221 * the guest CPU mode.
8222 *
8223 * @returns VBox status code.
8224 * @param pVM Pointer to the VM.
8225 * @param pVCpu Pointer to the VMCPU.
8226 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8227 * out-of-sync. Make sure to update the required fields
8228 * before using them.
8229 *
8230 * @remarks No-long-jump zone!!!
8231 */
8232static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8233{
8234 AssertPtr(pVM);
8235 AssertPtr(pVCpu);
8236 AssertPtr(pMixedCtx);
8237 HMVMX_ASSERT_PREEMPT_SAFE();
8238
8239 VMMRZCallRing3Disable(pVCpu);
8240 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8241
8242 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8243
8244 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8245
8246 /* Determine real-on-v86 mode. */
8247 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8248 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8249 && CPUMIsGuestInRealModeEx(pMixedCtx))
8250 {
8251 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8252 }
8253
8254 /*
8255 * Load the guest-state into the VMCS.
8256 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8257 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8258 */
8259 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8260 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8261
8262 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8263 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8264 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8265
8266 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8267 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8268 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8269
8270 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8271 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8272
8273 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8274 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8275
8276 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8277 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8278 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8279
8280 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8281 determine we don't have to swap EFER after all. */
8282 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8283 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8284
8285 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 /*
8289 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8290 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8291 */
8292 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8293 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8294
8295 /* Clear any unused and reserved bits. */
8296 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8297
8298 VMMRZCallRing3Enable(pVCpu);
8299
8300 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8301 return rc;
8302}
8303
8304
8305/**
8306 * Loads the state shared between the host and guest into the VMCS.
8307 *
8308 * @param pVM Pointer to the VM.
8309 * @param pVCpu Pointer to the VMCPU.
8310 * @param pCtx Pointer to the guest-CPU context.
8311 *
8312 * @remarks No-long-jump zone!!!
8313 */
8314static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8315{
8316 NOREF(pVM);
8317
8318 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8319 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8320
8321 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8322 {
8323 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8324 AssertRC(rc);
8325 }
8326
8327 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8328 {
8329 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8330 AssertRC(rc);
8331
8332 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8333 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8334 {
8335 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8336 AssertRC(rc);
8337 }
8338 }
8339
8340 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8341 {
8342#if HC_ARCH_BITS == 64
8343 if (pVM->hm.s.fAllow64BitGuests)
8344 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8345#endif
8346 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8347 }
8348
8349 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8350 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8351}
8352
8353
8354/**
8355 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8356 *
8357 * @param pVM Pointer to the VM.
8358 * @param pVCpu Pointer to the VMCPU.
8359 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8360 * out-of-sync. Make sure to update the required fields
8361 * before using them.
8362 */
8363DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8364{
8365 HMVMX_ASSERT_PREEMPT_SAFE();
8366
8367 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8368#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8369 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8370#endif
8371
8372 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8373 {
8374 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8375 AssertRC(rc);
8376 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8377 }
8378 else if (HMCPU_CF_VALUE(pVCpu))
8379 {
8380 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8381 AssertRC(rc);
8382 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8383 }
8384
8385 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8386 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8387 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8388 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8389}
8390
8391
8392/**
8393 * Does the preparations before executing guest code in VT-x.
8394 *
8395 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8396 * recompiler. We must be cautious what we do here regarding committing
8397 * guest-state information into the VMCS assuming we assuredly execute the
8398 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8399 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8400 * so that the recompiler can (and should) use them when it resumes guest
8401 * execution. Otherwise such operations must be done when we can no longer
8402 * exit to ring-3.
8403 *
8404 * @returns Strict VBox status code.
8405 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8406 * have been disabled.
8407 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8408 * double-fault into the guest.
8409 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8410 *
8411 * @param pVM Pointer to the VM.
8412 * @param pVCpu Pointer to the VMCPU.
8413 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8414 * out-of-sync. Make sure to update the required fields
8415 * before using them.
8416 * @param pVmxTransient Pointer to the VMX transient structure.
8417 */
8418static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8419{
8420 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8421
8422#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8423 PGMRZDynMapFlushAutoSet(pVCpu);
8424#endif
8425
8426 /* Check force flag actions that might require us to go back to ring-3. */
8427 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8428 if (rc != VINF_SUCCESS)
8429 return rc;
8430
8431#ifndef IEM_VERIFICATION_MODE_FULL
8432 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8433 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8434 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8435 {
8436 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8437 RTGCPHYS GCPhysApicBase;
8438 GCPhysApicBase = pMixedCtx->msrApicBase;
8439 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8440
8441 /* Unalias any existing mapping. */
8442 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8443 AssertRCReturn(rc, rc);
8444
8445 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8446 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8447 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8448 AssertRCReturn(rc, rc);
8449
8450 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8451 }
8452#endif /* !IEM_VERIFICATION_MODE_FULL */
8453
8454 if (TRPMHasTrap(pVCpu))
8455 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8456 else if (!pVCpu->hm.s.Event.fPending)
8457 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8458
8459 /*
8460 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8461 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8462 */
8463 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8464 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8465 {
8466 Assert(rc == VINF_EM_RESET);
8467 return rc;
8468 }
8469
8470 /*
8471 * Load the guest state bits, we can handle longjmps/getting preempted here.
8472 *
8473 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8474 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8475 * Hence, this needs to be done -after- injection of events.
8476 */
8477 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8478
8479 /*
8480 * No longjmps to ring-3 from this point on!!!
8481 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8482 * This also disables flushing of the R0-logger instance (if any).
8483 */
8484 VMMRZCallRing3Disable(pVCpu);
8485
8486 /*
8487 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8488 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8489 *
8490 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8491 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8492 *
8493 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8494 * executing guest code.
8495 */
8496 pVmxTransient->uEflags = ASMIntDisableFlags();
8497 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8498 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8499 {
8500 hmR0VmxClearEventVmcs(pVCpu);
8501 ASMSetFlags(pVmxTransient->uEflags);
8502 VMMRZCallRing3Enable(pVCpu);
8503 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8504 return VINF_EM_RAW_TO_R3;
8505 }
8506
8507 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8508 {
8509 hmR0VmxClearEventVmcs(pVCpu);
8510 ASMSetFlags(pVmxTransient->uEflags);
8511 VMMRZCallRing3Enable(pVCpu);
8512 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8513 return VINF_EM_RAW_INTERRUPT;
8514 }
8515
8516 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8517 pVCpu->hm.s.Event.fPending = false;
8518
8519 return VINF_SUCCESS;
8520}
8521
8522
8523/**
8524 * Prepares to run guest code in VT-x and we've committed to doing so. This
8525 * means there is no backing out to ring-3 or anywhere else at this
8526 * point.
8527 *
8528 * @param pVM Pointer to the VM.
8529 * @param pVCpu Pointer to the VMCPU.
8530 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8531 * out-of-sync. Make sure to update the required fields
8532 * before using them.
8533 * @param pVmxTransient Pointer to the VMX transient structure.
8534 *
8535 * @remarks Called with preemption disabled.
8536 * @remarks No-long-jump zone!!!
8537 */
8538static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8539{
8540 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8541 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8542 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8543
8544 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8545 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8546
8547#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8548 if (!CPUMIsGuestFPUStateActive(pVCpu))
8549 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8550 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8551#endif
8552
8553 if ( pVCpu->hm.s.fUseGuestFpu
8554 && !CPUMIsGuestFPUStateActive(pVCpu))
8555 {
8556 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8557 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8558 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8559 }
8560
8561 /*
8562 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8563 */
8564 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8565 && pVCpu->hm.s.vmx.cMsrs > 0)
8566 {
8567 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8568 }
8569
8570 /*
8571 * Load the host state bits as we may've been preempted (only happens when
8572 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8573 */
8574 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8575 {
8576 /* This ASSUMES that pfnStartVM has been set up already. */
8577 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8578 AssertRC(rc);
8579 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8580 }
8581 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8582
8583 /*
8584 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8585 */
8586 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8587 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8588 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8589
8590 /* Store status of the shared guest-host state at the time of VM-entry. */
8591#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8592 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8593 {
8594 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8595 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8596 }
8597 else
8598#endif
8599 {
8600 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8601 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8602 }
8603 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8604
8605 /*
8606 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8607 */
8608 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8609 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8610
8611 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8612 RTCPUID idCurrentCpu = pCpu->idCpu;
8613 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8614 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8615 {
8616 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8617 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8618 }
8619
8620 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8621 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8622 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8623 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8624
8625 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8626
8627 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8628 to start executing. */
8629
8630 /*
8631 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8632 */
8633 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8634 {
8635 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8636 {
8637 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8638 AssertRC(rc2);
8639 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8640 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8641 true /* fUpdateHostMsr */);
8642 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8643 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8644 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8645 }
8646 else
8647 {
8648 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8649 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8650 }
8651 }
8652
8653#ifdef VBOX_STRICT
8654 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8655 hmR0VmxCheckHostEferMsr(pVCpu);
8656 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8657#endif
8658#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8659 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8660 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8661 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8662#endif
8663}
8664
8665
8666/**
8667 * Performs some essential restoration of state after running guest code in
8668 * VT-x.
8669 *
8670 * @param pVM Pointer to the VM.
8671 * @param pVCpu Pointer to the VMCPU.
8672 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8673 * out-of-sync. Make sure to update the required fields
8674 * before using them.
8675 * @param pVmxTransient Pointer to the VMX transient structure.
8676 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8677 *
8678 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8679 *
8680 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8681 * unconditionally when it is safe to do so.
8682 */
8683static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8684{
8685 NOREF(pVM);
8686
8687 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8688
8689 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8690 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8691 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8692 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8693 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8694 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8695
8696 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8697 {
8698 /** @todo Find a way to fix hardcoding a guestimate. */
8699 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8700 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8701 }
8702
8703 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8704 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8705 Assert(!(ASMGetFlags() & X86_EFL_IF));
8706 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8707
8708#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8709 if (CPUMIsGuestFPUStateActive(pVCpu))
8710 {
8711 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8712 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8713 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8714 }
8715#endif
8716
8717#if HC_ARCH_BITS == 64
8718 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8719#endif
8720 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8721#ifdef VBOX_STRICT
8722 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8723#endif
8724 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8725 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8726
8727 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8728 uint32_t uExitReason;
8729 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8730 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8731 AssertRC(rc);
8732 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8733 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8734
8735 /* Update the VM-exit history array. */
8736 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8737
8738 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8739 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8740 {
8741 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8742 pVmxTransient->fVMEntryFailed));
8743 return;
8744 }
8745
8746 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8747 {
8748 /** @todo We can optimize this by only syncing with our force-flags when
8749 * really needed and keeping the VMCS state as it is for most
8750 * VM-exits. */
8751 /* Update the guest interruptibility-state from the VMCS. */
8752 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8753
8754#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8755 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8756 AssertRC(rc);
8757#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8758 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8759 AssertRC(rc);
8760#endif
8761
8762 /*
8763 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8764 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8765 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8766 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8767 */
8768 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8769 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8770 {
8771 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8772 AssertRC(rc);
8773 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8774 }
8775 }
8776}
8777
8778
8779/**
8780 * Runs the guest code using VT-x the normal way.
8781 *
8782 * @returns VBox status code.
8783 * @param pVM Pointer to the VM.
8784 * @param pVCpu Pointer to the VMCPU.
8785 * @param pCtx Pointer to the guest-CPU context.
8786 *
8787 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8788 */
8789static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8790{
8791 VMXTRANSIENT VmxTransient;
8792 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8793 int rc = VERR_INTERNAL_ERROR_5;
8794 uint32_t cLoops = 0;
8795
8796 for (;; cLoops++)
8797 {
8798 Assert(!HMR0SuspendPending());
8799 HMVMX_ASSERT_CPU_SAFE();
8800
8801 /* Preparatory work for running guest code, this may force us to return
8802 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8803 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8804 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8805 if (rc != VINF_SUCCESS)
8806 break;
8807
8808 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8809 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8810 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8811
8812 /* Restore any residual host-state and save any bits shared between host
8813 and guest into the guest-CPU state. Re-enables interrupts! */
8814 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8815
8816 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8817 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8818 {
8819 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8820 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8821 return rc;
8822 }
8823
8824 /* Handle the VM-exit. */
8825 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8827 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8828 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8829 HMVMX_START_EXIT_DISPATCH_PROF();
8830#ifdef HMVMX_USE_FUNCTION_TABLE
8831 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8832#else
8833 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8834#endif
8835 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8836 if (rc != VINF_SUCCESS)
8837 break;
8838 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8839 {
8840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8841 rc = VINF_EM_RAW_INTERRUPT;
8842 break;
8843 }
8844 }
8845
8846 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8847 return rc;
8848}
8849
8850
8851/**
8852 * Single steps guest code using VT-x.
8853 *
8854 * @returns VBox status code.
8855 * @param pVM Pointer to the VM.
8856 * @param pVCpu Pointer to the VMCPU.
8857 * @param pCtx Pointer to the guest-CPU context.
8858 *
8859 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8860 */
8861static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8862{
8863 VMXTRANSIENT VmxTransient;
8864 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8865 int rc = VERR_INTERNAL_ERROR_5;
8866 uint32_t cLoops = 0;
8867 uint16_t uCsStart = pCtx->cs.Sel;
8868 uint64_t uRipStart = pCtx->rip;
8869
8870 for (;; cLoops++)
8871 {
8872 Assert(!HMR0SuspendPending());
8873 HMVMX_ASSERT_CPU_SAFE();
8874
8875 /* Preparatory work for running guest code, this may force us to return
8876 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8877 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8878 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8879 if (rc != VINF_SUCCESS)
8880 break;
8881
8882 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8883 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8884 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8885
8886 /* Restore any residual host-state and save any bits shared between host
8887 and guest into the guest-CPU state. Re-enables interrupts! */
8888 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8889
8890 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8891 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8892 {
8893 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8894 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8895 return rc;
8896 }
8897
8898 /* Handle the VM-exit. */
8899 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8900 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8901 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8902 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8903 HMVMX_START_EXIT_DISPATCH_PROF();
8904#ifdef HMVMX_USE_FUNCTION_TABLE
8905 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8906#else
8907 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8908#endif
8909 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8910 if (rc != VINF_SUCCESS)
8911 break;
8912 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8913 {
8914 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8915 rc = VINF_EM_RAW_INTERRUPT;
8916 break;
8917 }
8918
8919 /*
8920 * Did the RIP change, if so, consider it a single step.
8921 * Otherwise, make sure one of the TFs gets set.
8922 */
8923 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8924 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8925 AssertRCReturn(rc2, rc2);
8926 if ( pCtx->rip != uRipStart
8927 || pCtx->cs.Sel != uCsStart)
8928 {
8929 rc = VINF_EM_DBG_STEPPED;
8930 break;
8931 }
8932 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8933 }
8934
8935 /*
8936 * Clear the X86_EFL_TF if necessary.
8937 */
8938 if (pVCpu->hm.s.fClearTrapFlag)
8939 {
8940 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8941 AssertRCReturn(rc2, rc2);
8942 pVCpu->hm.s.fClearTrapFlag = false;
8943 pCtx->eflags.Bits.u1TF = 0;
8944 }
8945 /** @todo there seems to be issues with the resume flag when the monitor trap
8946 * flag is pending without being used. Seen early in bios init when
8947 * accessing APIC page in protected mode. */
8948
8949 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8950 return rc;
8951}
8952
8953
8954/**
8955 * Runs the guest code using VT-x.
8956 *
8957 * @returns VBox status code.
8958 * @param pVM Pointer to the VM.
8959 * @param pVCpu Pointer to the VMCPU.
8960 * @param pCtx Pointer to the guest-CPU context.
8961 */
8962VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8963{
8964 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8965 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8966 HMVMX_ASSERT_PREEMPT_SAFE();
8967
8968 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8969
8970 int rc;
8971 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8972 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8973 else
8974 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8975
8976 if (rc == VERR_EM_INTERPRETER)
8977 rc = VINF_EM_RAW_EMULATE_INSTR;
8978 else if (rc == VINF_EM_RESET)
8979 rc = VINF_EM_TRIPLE_FAULT;
8980
8981 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8982 if (RT_FAILURE(rc2))
8983 {
8984 pVCpu->hm.s.u32HMError = rc;
8985 rc = rc2;
8986 }
8987 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8988 return rc;
8989}
8990
8991
8992#ifndef HMVMX_USE_FUNCTION_TABLE
8993DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8994{
8995#ifdef DEBUG_ramshankar
8996# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8997# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8998#endif
8999 int rc;
9000 switch (rcReason)
9001 {
9002 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9003 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9004 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9005 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9006 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9007 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9008 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9009 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9010 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9011 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9012 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9013 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036
9037 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9038 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9039 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9040 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9041 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9042 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9043 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9044 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9045 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9046
9047 case VMX_EXIT_VMCLEAR:
9048 case VMX_EXIT_VMLAUNCH:
9049 case VMX_EXIT_VMPTRLD:
9050 case VMX_EXIT_VMPTRST:
9051 case VMX_EXIT_VMREAD:
9052 case VMX_EXIT_VMRESUME:
9053 case VMX_EXIT_VMWRITE:
9054 case VMX_EXIT_VMXOFF:
9055 case VMX_EXIT_VMXON:
9056 case VMX_EXIT_INVEPT:
9057 case VMX_EXIT_INVVPID:
9058 case VMX_EXIT_VMFUNC:
9059 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9060 break;
9061 default:
9062 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9063 break;
9064 }
9065 return rc;
9066}
9067#endif
9068
9069#ifdef DEBUG
9070/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9071# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9072 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9073
9074# define HMVMX_ASSERT_PREEMPT_CPUID() \
9075 do \
9076 { \
9077 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9078 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9079 } while (0)
9080
9081# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9082 do { \
9083 AssertPtr(pVCpu); \
9084 AssertPtr(pMixedCtx); \
9085 AssertPtr(pVmxTransient); \
9086 Assert(pVmxTransient->fVMEntryFailed == false); \
9087 Assert(ASMIntAreEnabled()); \
9088 HMVMX_ASSERT_PREEMPT_SAFE(); \
9089 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9090 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9091 HMVMX_ASSERT_PREEMPT_SAFE(); \
9092 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9093 HMVMX_ASSERT_PREEMPT_CPUID(); \
9094 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9095 } while (0)
9096
9097# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9098 do { \
9099 Log4Func(("\n")); \
9100 } while (0)
9101#else /* Release builds */
9102# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9103 do { \
9104 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9105 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9106 } while (0)
9107# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9108#endif
9109
9110
9111/**
9112 * Advances the guest RIP after reading it from the VMCS.
9113 *
9114 * @returns VBox status code.
9115 * @param pVCpu Pointer to the VMCPU.
9116 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9117 * out-of-sync. Make sure to update the required fields
9118 * before using them.
9119 * @param pVmxTransient Pointer to the VMX transient structure.
9120 *
9121 * @remarks No-long-jump zone!!!
9122 */
9123DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9124{
9125 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9126 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9127 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9128 AssertRCReturn(rc, rc);
9129
9130 pMixedCtx->rip += pVmxTransient->cbInstr;
9131 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9132
9133 /*
9134 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9135 * pending debug exception field as it takes care of priority of events.
9136 *
9137 * See Intel spec. 32.2.1 "Debug Exceptions".
9138 */
9139 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9140
9141 return rc;
9142}
9143
9144
9145/**
9146 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9147 * and update error record fields accordingly.
9148 *
9149 * @return VMX_IGS_* return codes.
9150 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9151 * wrong with the guest state.
9152 *
9153 * @param pVM Pointer to the VM.
9154 * @param pVCpu Pointer to the VMCPU.
9155 * @param pCtx Pointer to the guest-CPU state.
9156 *
9157 * @remarks This function assumes our cache of the VMCS controls
9158 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9159 */
9160static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9161{
9162#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9163#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9164 uError = (err); \
9165 break; \
9166 } else do { } while (0)
9167
9168 int rc;
9169 uint32_t uError = VMX_IGS_ERROR;
9170 uint32_t u32Val;
9171 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9172
9173 do
9174 {
9175 /*
9176 * CR0.
9177 */
9178 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9179 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9180 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9181 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9182 if (fUnrestrictedGuest)
9183 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9184
9185 uint32_t u32GuestCR0;
9186 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9187 AssertRCBreak(rc);
9188 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9189 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9190 if ( !fUnrestrictedGuest
9191 && (u32GuestCR0 & X86_CR0_PG)
9192 && !(u32GuestCR0 & X86_CR0_PE))
9193 {
9194 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9195 }
9196
9197 /*
9198 * CR4.
9199 */
9200 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9201 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9202
9203 uint32_t u32GuestCR4;
9204 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9205 AssertRCBreak(rc);
9206 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9207 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9208
9209 /*
9210 * IA32_DEBUGCTL MSR.
9211 */
9212 uint64_t u64Val;
9213 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9214 AssertRCBreak(rc);
9215 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9216 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9217 {
9218 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9219 }
9220 uint64_t u64DebugCtlMsr = u64Val;
9221
9222#ifdef VBOX_STRICT
9223 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9224 AssertRCBreak(rc);
9225 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9226#endif
9227 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9228
9229 /*
9230 * RIP and RFLAGS.
9231 */
9232 uint32_t u32Eflags;
9233#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9234 if (HMVMX_IS_64BIT_HOST_MODE())
9235 {
9236 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9237 AssertRCBreak(rc);
9238 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9239 if ( !fLongModeGuest
9240 || !pCtx->cs.Attr.n.u1Long)
9241 {
9242 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9243 }
9244 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9245 * must be identical if the "IA-32e mode guest" VM-entry
9246 * control is 1 and CS.L is 1. No check applies if the
9247 * CPU supports 64 linear-address bits. */
9248
9249 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9250 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9251 AssertRCBreak(rc);
9252 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9253 VMX_IGS_RFLAGS_RESERVED);
9254 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9255 u32Eflags = u64Val;
9256 }
9257 else
9258#endif
9259 {
9260 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9261 AssertRCBreak(rc);
9262 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9263 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9264 }
9265
9266 if ( fLongModeGuest
9267 || ( fUnrestrictedGuest
9268 && !(u32GuestCR0 & X86_CR0_PE)))
9269 {
9270 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9271 }
9272
9273 uint32_t u32EntryInfo;
9274 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9275 AssertRCBreak(rc);
9276 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9277 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9278 {
9279 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9280 }
9281
9282 /*
9283 * 64-bit checks.
9284 */
9285#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9286 if (HMVMX_IS_64BIT_HOST_MODE())
9287 {
9288 if ( fLongModeGuest
9289 && !fUnrestrictedGuest)
9290 {
9291 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9292 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9293 }
9294
9295 if ( !fLongModeGuest
9296 && (u32GuestCR4 & X86_CR4_PCIDE))
9297 {
9298 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9299 }
9300
9301 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9302 * 51:32 beyond the processor's physical-address width are 0. */
9303
9304 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9305 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9306 {
9307 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9308 }
9309
9310 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9311 AssertRCBreak(rc);
9312 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9313
9314 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9315 AssertRCBreak(rc);
9316 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9317 }
9318#endif
9319
9320 /*
9321 * PERF_GLOBAL MSR.
9322 */
9323 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9324 {
9325 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9326 AssertRCBreak(rc);
9327 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9328 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9329 }
9330
9331 /*
9332 * PAT MSR.
9333 */
9334 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9335 {
9336 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9337 AssertRCBreak(rc);
9338 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9339 for (unsigned i = 0; i < 8; i++)
9340 {
9341 uint8_t u8Val = (u64Val & 0x7);
9342 if ( u8Val != 0 /* UC */
9343 || u8Val != 1 /* WC */
9344 || u8Val != 4 /* WT */
9345 || u8Val != 5 /* WP */
9346 || u8Val != 6 /* WB */
9347 || u8Val != 7 /* UC- */)
9348 {
9349 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9350 }
9351 u64Val >>= 3;
9352 }
9353 }
9354
9355 /*
9356 * EFER MSR.
9357 */
9358 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9359 {
9360 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9361 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9362 AssertRCBreak(rc);
9363 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9364 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9365 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9366 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9367 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9368 || !(u32GuestCR0 & X86_CR0_PG)
9369 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9370 VMX_IGS_EFER_LMA_LME_MISMATCH);
9371 }
9372
9373 /*
9374 * Segment registers.
9375 */
9376 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9377 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9378 if (!(u32Eflags & X86_EFL_VM))
9379 {
9380 /* CS */
9381 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9382 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9383 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9384 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9385 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9386 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9387 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9388 /* CS cannot be loaded with NULL in protected mode. */
9389 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9390 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9391 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9392 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9393 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9394 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9395 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9396 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9397 else
9398 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9399
9400 /* SS */
9401 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9402 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9403 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9404 if ( !(pCtx->cr0 & X86_CR0_PE)
9405 || pCtx->cs.Attr.n.u4Type == 3)
9406 {
9407 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9408 }
9409 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9410 {
9411 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9412 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9413 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9414 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9415 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9416 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9417 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9418 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9419 }
9420
9421 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9422 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9423 {
9424 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9425 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9426 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9427 || pCtx->ds.Attr.n.u4Type > 11
9428 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9429 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9430 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9431 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9432 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9433 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9434 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9435 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9436 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9437 }
9438 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9439 {
9440 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9441 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9442 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9443 || pCtx->es.Attr.n.u4Type > 11
9444 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9445 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9446 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9447 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9448 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9449 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9450 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9451 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9452 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9453 }
9454 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9455 {
9456 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9457 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9458 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9459 || pCtx->fs.Attr.n.u4Type > 11
9460 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9461 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9462 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9463 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9464 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9465 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9466 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9467 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9468 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9469 }
9470 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9471 {
9472 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9473 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9474 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9475 || pCtx->gs.Attr.n.u4Type > 11
9476 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9477 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9478 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9479 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9480 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9481 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9482 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9483 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9484 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9485 }
9486 /* 64-bit capable CPUs. */
9487#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9488 if (HMVMX_IS_64BIT_HOST_MODE())
9489 {
9490 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9491 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9492 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9493 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9494 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9495 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9496 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9497 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9498 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9499 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9500 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9501 }
9502#endif
9503 }
9504 else
9505 {
9506 /* V86 mode checks. */
9507 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9508 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9509 {
9510 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9511 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9512 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9513 }
9514 else
9515 {
9516 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9517 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9518 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9519 }
9520
9521 /* CS */
9522 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9523 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9524 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9525 /* SS */
9526 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9527 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9528 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9529 /* DS */
9530 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9531 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9532 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9533 /* ES */
9534 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9535 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9536 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9537 /* FS */
9538 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9539 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9540 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9541 /* GS */
9542 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9543 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9544 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9545 /* 64-bit capable CPUs. */
9546#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9547 if (HMVMX_IS_64BIT_HOST_MODE())
9548 {
9549 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9550 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9551 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9552 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9553 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9554 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9555 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9556 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9557 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9558 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9559 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9560 }
9561#endif
9562 }
9563
9564 /*
9565 * TR.
9566 */
9567 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9568 /* 64-bit capable CPUs. */
9569#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9570 if (HMVMX_IS_64BIT_HOST_MODE())
9571 {
9572 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9573 }
9574#endif
9575 if (fLongModeGuest)
9576 {
9577 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9578 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9579 }
9580 else
9581 {
9582 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9583 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9584 VMX_IGS_TR_ATTR_TYPE_INVALID);
9585 }
9586 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9587 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9588 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9589 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9590 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9591 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9592 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9593 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9594
9595 /*
9596 * GDTR and IDTR.
9597 */
9598#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9599 if (HMVMX_IS_64BIT_HOST_MODE())
9600 {
9601 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9602 AssertRCBreak(rc);
9603 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9604
9605 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9606 AssertRCBreak(rc);
9607 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9608 }
9609#endif
9610
9611 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9612 AssertRCBreak(rc);
9613 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9614
9615 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9616 AssertRCBreak(rc);
9617 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9618
9619 /*
9620 * Guest Non-Register State.
9621 */
9622 /* Activity State. */
9623 uint32_t u32ActivityState;
9624 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9625 AssertRCBreak(rc);
9626 HMVMX_CHECK_BREAK( !u32ActivityState
9627 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9628 VMX_IGS_ACTIVITY_STATE_INVALID);
9629 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9630 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9631 uint32_t u32IntrState;
9632 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9633 AssertRCBreak(rc);
9634 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9635 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9636 {
9637 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9638 }
9639
9640 /** @todo Activity state and injecting interrupts. Left as a todo since we
9641 * currently don't use activity states but ACTIVE. */
9642
9643 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9644 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9645
9646 /* Guest interruptibility-state. */
9647 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9648 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9649 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9650 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9651 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9652 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9653 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9654 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9655 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9656 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9657 {
9658 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9659 {
9660 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9661 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9662 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9663 }
9664 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9665 {
9666 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9667 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9668 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9669 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9670 }
9671 }
9672 /** @todo Assumes the processor is not in SMM. */
9673 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9674 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9675 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9676 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9677 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9678 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9679 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9680 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9681 {
9682 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9683 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9684 }
9685
9686 /* Pending debug exceptions. */
9687 if (HMVMX_IS_64BIT_HOST_MODE())
9688 {
9689 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9690 AssertRCBreak(rc);
9691 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9692 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9693 u32Val = u64Val; /* For pending debug exceptions checks below. */
9694 }
9695 else
9696 {
9697 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9698 AssertRCBreak(rc);
9699 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9700 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9701 }
9702
9703 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9704 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9705 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9706 {
9707 if ( (u32Eflags & X86_EFL_TF)
9708 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9709 {
9710 /* Bit 14 is PendingDebug.BS. */
9711 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9712 }
9713 if ( !(u32Eflags & X86_EFL_TF)
9714 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9715 {
9716 /* Bit 14 is PendingDebug.BS. */
9717 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9718 }
9719 }
9720
9721 /* VMCS link pointer. */
9722 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9723 AssertRCBreak(rc);
9724 if (u64Val != UINT64_C(0xffffffffffffffff))
9725 {
9726 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9727 /** @todo Bits beyond the processor's physical-address width MBZ. */
9728 /** @todo 32-bit located in memory referenced by value of this field (as a
9729 * physical address) must contain the processor's VMCS revision ID. */
9730 /** @todo SMM checks. */
9731 }
9732
9733 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9734 * not using Nested Paging? */
9735 if ( pVM->hm.s.fNestedPaging
9736 && !fLongModeGuest
9737 && CPUMIsGuestInPAEModeEx(pCtx))
9738 {
9739 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9740 AssertRCBreak(rc);
9741 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9742
9743 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9744 AssertRCBreak(rc);
9745 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9746
9747 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9748 AssertRCBreak(rc);
9749 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9750
9751 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9752 AssertRCBreak(rc);
9753 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9754 }
9755
9756 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9757 if (uError == VMX_IGS_ERROR)
9758 uError = VMX_IGS_REASON_NOT_FOUND;
9759 } while (0);
9760
9761 pVCpu->hm.s.u32HMError = uError;
9762 return uError;
9763
9764#undef HMVMX_ERROR_BREAK
9765#undef HMVMX_CHECK_BREAK
9766}
9767
9768/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9769/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9770/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9771
9772/** @name VM-exit handlers.
9773 * @{
9774 */
9775
9776/**
9777 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9778 */
9779HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9780{
9781 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9782 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9783 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9784 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9785 return VINF_SUCCESS;
9786 return VINF_EM_RAW_INTERRUPT;
9787}
9788
9789
9790/**
9791 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9792 */
9793HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9794{
9795 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9796 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9797
9798 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9799 AssertRCReturn(rc, rc);
9800
9801 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9802 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9803 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9804 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9805
9806 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9807 {
9808 /*
9809 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9810 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9811 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9812 *
9813 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9814 */
9815 VMXDispatchHostNmi();
9816 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9817 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9818 return VINF_SUCCESS;
9819 }
9820
9821 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9822 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9823 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9824 {
9825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9826 return VINF_SUCCESS;
9827 }
9828 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9829 {
9830 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9831 return rc;
9832 }
9833
9834 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9835 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9836 switch (uIntType)
9837 {
9838 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9839 Assert(uVector == X86_XCPT_DB);
9840 /* no break */
9841 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9842 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9843 /* no break */
9844 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9845 {
9846 switch (uVector)
9847 {
9848 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9849 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9850 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9851 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9852 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9853 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9854#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9855 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9856 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9857 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9858 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9859 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9860 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9861 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9862 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9863 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9864 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9865 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9866 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9867#endif
9868 default:
9869 {
9870 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9871 AssertRCReturn(rc, rc);
9872
9873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9874 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9875 {
9876 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9877 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9878 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9879
9880 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9881 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9882 AssertRCReturn(rc, rc);
9883 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9884 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9885 0 /* GCPtrFaultAddress */);
9886 AssertRCReturn(rc, rc);
9887 }
9888 else
9889 {
9890 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9891 pVCpu->hm.s.u32HMError = uVector;
9892 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9893 }
9894 break;
9895 }
9896 }
9897 break;
9898 }
9899
9900 default:
9901 {
9902 pVCpu->hm.s.u32HMError = uExitIntInfo;
9903 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9904 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9905 break;
9906 }
9907 }
9908 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9909 return rc;
9910}
9911
9912
9913/**
9914 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9915 */
9916HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9917{
9918 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9919
9920 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9921 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9922
9923 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9924 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9925 return VINF_SUCCESS;
9926}
9927
9928
9929/**
9930 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9931 */
9932HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9933{
9934 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9935 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9936 {
9937 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9938 HMVMX_RETURN_UNEXPECTED_EXIT();
9939 }
9940
9941 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9942
9943 /*
9944 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9945 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9946 */
9947 uint32_t uIntrState = 0;
9948 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9949 AssertRCReturn(rc, rc);
9950
9951 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9952 if ( fBlockSti
9953 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9954 {
9955 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9956 }
9957
9958 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9959 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9960
9961 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9962 return VINF_SUCCESS;
9963}
9964
9965
9966/**
9967 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9968 */
9969HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9970{
9971 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9972 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9973 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9974}
9975
9976
9977/**
9978 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9979 */
9980HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9981{
9982 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9983 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9984 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9985}
9986
9987
9988/**
9989 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9990 */
9991HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9992{
9993 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9994 PVM pVM = pVCpu->CTX_SUFF(pVM);
9995 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9996 if (RT_LIKELY(rc == VINF_SUCCESS))
9997 {
9998 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9999 Assert(pVmxTransient->cbInstr == 2);
10000 }
10001 else
10002 {
10003 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10004 rc = VERR_EM_INTERPRETER;
10005 }
10006 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10007 return rc;
10008}
10009
10010
10011/**
10012 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10013 */
10014HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10015{
10016 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10017 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10018 AssertRCReturn(rc, rc);
10019
10020 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10021 return VINF_EM_RAW_EMULATE_INSTR;
10022
10023 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10024 HMVMX_RETURN_UNEXPECTED_EXIT();
10025}
10026
10027
10028/**
10029 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10030 */
10031HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10032{
10033 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10034 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10035 AssertRCReturn(rc, rc);
10036
10037 PVM pVM = pVCpu->CTX_SUFF(pVM);
10038 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10039 if (RT_LIKELY(rc == VINF_SUCCESS))
10040 {
10041 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10042 Assert(pVmxTransient->cbInstr == 2);
10043 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10044 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10045 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10046 }
10047 else
10048 {
10049 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
10050 rc = VERR_EM_INTERPRETER;
10051 }
10052 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10053 return rc;
10054}
10055
10056
10057/**
10058 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10059 */
10060HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10061{
10062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10063 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10064 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10065 AssertRCReturn(rc, rc);
10066
10067 PVM pVM = pVCpu->CTX_SUFF(pVM);
10068 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10069 if (RT_LIKELY(rc == VINF_SUCCESS))
10070 {
10071 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10072 Assert(pVmxTransient->cbInstr == 3);
10073 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10074 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10075 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10076 }
10077 else
10078 {
10079 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10080 rc = VERR_EM_INTERPRETER;
10081 }
10082 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10083 return rc;
10084}
10085
10086
10087/**
10088 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10089 */
10090HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10091{
10092 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10093 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10094 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10095 AssertRCReturn(rc, rc);
10096
10097 PVM pVM = pVCpu->CTX_SUFF(pVM);
10098 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10099 if (RT_LIKELY(rc == VINF_SUCCESS))
10100 {
10101 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10102 Assert(pVmxTransient->cbInstr == 2);
10103 }
10104 else
10105 {
10106 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10107 rc = VERR_EM_INTERPRETER;
10108 }
10109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10110 return rc;
10111}
10112
10113
10114/**
10115 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10116 */
10117HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10118{
10119 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10120
10121 int rc = VERR_NOT_SUPPORTED;
10122 if (GIMAreHypercallsEnabled(pVCpu))
10123 {
10124 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10125 AssertRCReturn(rc, rc);
10126
10127 rc = GIMHypercall(pVCpu, pMixedCtx);
10128 }
10129 if (rc != VINF_SUCCESS)
10130 {
10131 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10132 rc = VINF_SUCCESS;
10133 }
10134
10135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10136 return rc;
10137}
10138
10139
10140/**
10141 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10142 */
10143HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10144{
10145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10146 PVM pVM = pVCpu->CTX_SUFF(pVM);
10147 Assert(!pVM->hm.s.fNestedPaging);
10148
10149 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10150 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10151 AssertRCReturn(rc, rc);
10152
10153 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10154 rc = VBOXSTRICTRC_VAL(rc2);
10155 if (RT_LIKELY(rc == VINF_SUCCESS))
10156 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10157 else
10158 {
10159 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10160 pVmxTransient->uExitQualification, rc));
10161 }
10162 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10163 return rc;
10164}
10165
10166
10167/**
10168 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10169 */
10170HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10171{
10172 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10173 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10174 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10175 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10176 AssertRCReturn(rc, rc);
10177
10178 PVM pVM = pVCpu->CTX_SUFF(pVM);
10179 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10180 if (RT_LIKELY(rc == VINF_SUCCESS))
10181 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10182 else
10183 {
10184 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10185 rc = VERR_EM_INTERPRETER;
10186 }
10187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10188 return rc;
10189}
10190
10191
10192/**
10193 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10194 */
10195HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10196{
10197 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10198 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10199 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10200 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10201 AssertRCReturn(rc, rc);
10202
10203 PVM pVM = pVCpu->CTX_SUFF(pVM);
10204 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10205 rc = VBOXSTRICTRC_VAL(rc2);
10206 if (RT_LIKELY( rc == VINF_SUCCESS
10207 || rc == VINF_EM_HALT))
10208 {
10209 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10210 AssertRCReturn(rc3, rc3);
10211
10212 if ( rc == VINF_EM_HALT
10213 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10214 {
10215 rc = VINF_SUCCESS;
10216 }
10217 }
10218 else
10219 {
10220 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10221 rc = VERR_EM_INTERPRETER;
10222 }
10223 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10224 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10225 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10226 return rc;
10227}
10228
10229
10230/**
10231 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10232 */
10233HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10234{
10235 /*
10236 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10237 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10238 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10239 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10240 */
10241 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10242 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10243 HMVMX_RETURN_UNEXPECTED_EXIT();
10244}
10245
10246
10247/**
10248 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10249 */
10250HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10251{
10252 /*
10253 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10254 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10255 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10256 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10257 */
10258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10259 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10260 HMVMX_RETURN_UNEXPECTED_EXIT();
10261}
10262
10263
10264/**
10265 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10266 */
10267HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10268{
10269 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10270 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10271 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10272 HMVMX_RETURN_UNEXPECTED_EXIT();
10273}
10274
10275
10276/**
10277 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10278 */
10279HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10280{
10281 /*
10282 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10283 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10284 * See Intel spec. 25.3 "Other Causes of VM-exits".
10285 */
10286 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10287 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10288 HMVMX_RETURN_UNEXPECTED_EXIT();
10289}
10290
10291
10292/**
10293 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10294 * VM-exit.
10295 */
10296HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10297{
10298 /*
10299 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10300 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10301 *
10302 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10303 * See Intel spec. "23.8 Restrictions on VMX operation".
10304 */
10305 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10306 return VINF_SUCCESS;
10307}
10308
10309
10310/**
10311 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10312 * VM-exit.
10313 */
10314HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10315{
10316 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10317 return VINF_EM_RESET;
10318}
10319
10320
10321/**
10322 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10323 */
10324HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10325{
10326 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10327 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10328 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10329 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10330 AssertRCReturn(rc, rc);
10331
10332 pMixedCtx->rip++;
10333 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10334 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10335 rc = VINF_SUCCESS;
10336 else
10337 rc = VINF_EM_HALT;
10338
10339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10340 return rc;
10341}
10342
10343
10344/**
10345 * VM-exit handler for instructions that result in a #UD exception delivered to
10346 * the guest.
10347 */
10348HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10349{
10350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10351 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10352 return VINF_SUCCESS;
10353}
10354
10355
10356/**
10357 * VM-exit handler for expiry of the VMX preemption timer.
10358 */
10359HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10360{
10361 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10362
10363 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10364 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10365
10366 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10367 PVM pVM = pVCpu->CTX_SUFF(pVM);
10368 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10369 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10370 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10371}
10372
10373
10374/**
10375 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10376 */
10377HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10378{
10379 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10380
10381 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10382 /** @todo check if XSETBV is supported by the recompiler. */
10383 return VERR_EM_INTERPRETER;
10384}
10385
10386
10387/**
10388 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10389 */
10390HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10391{
10392 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10393
10394 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10395 /** @todo implement EMInterpretInvpcid() */
10396 return VERR_EM_INTERPRETER;
10397}
10398
10399
10400/**
10401 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10402 * Error VM-exit.
10403 */
10404HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10405{
10406 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10407 AssertRCReturn(rc, rc);
10408
10409 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10410 AssertRCReturn(rc, rc);
10411
10412 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10413 NOREF(uInvalidReason);
10414
10415#ifdef VBOX_STRICT
10416 uint32_t uIntrState;
10417 HMVMXHCUINTREG uHCReg;
10418 uint64_t u64Val;
10419 uint32_t u32Val;
10420
10421 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10422 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10423 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10424 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10425 AssertRCReturn(rc, rc);
10426
10427 Log4(("uInvalidReason %u\n", uInvalidReason));
10428 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10429 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10430 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10431 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10432
10433 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10434 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10435 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10436 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10437 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10438 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10439 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10440 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10441 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10442 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10443 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10444 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10445#else
10446 NOREF(pVmxTransient);
10447#endif
10448
10449 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10450 return VERR_VMX_INVALID_GUEST_STATE;
10451}
10452
10453
10454/**
10455 * VM-exit handler for VM-entry failure due to an MSR-load
10456 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10457 */
10458HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10459{
10460 NOREF(pVmxTransient);
10461 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10462 HMVMX_RETURN_UNEXPECTED_EXIT();
10463}
10464
10465
10466/**
10467 * VM-exit handler for VM-entry failure due to a machine-check event
10468 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10469 */
10470HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10471{
10472 NOREF(pVmxTransient);
10473 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10474 HMVMX_RETURN_UNEXPECTED_EXIT();
10475}
10476
10477
10478/**
10479 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10480 * theory.
10481 */
10482HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10483{
10484 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10485 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10486 return VERR_VMX_UNDEFINED_EXIT_CODE;
10487}
10488
10489
10490/**
10491 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10492 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10493 * Conditional VM-exit.
10494 */
10495HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10496{
10497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10498
10499 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10500 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10501 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10502 return VERR_EM_INTERPRETER;
10503 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10504 HMVMX_RETURN_UNEXPECTED_EXIT();
10505}
10506
10507
10508/**
10509 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10510 */
10511HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10512{
10513 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10514
10515 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10516 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10517 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10518 return VERR_EM_INTERPRETER;
10519 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10520 HMVMX_RETURN_UNEXPECTED_EXIT();
10521}
10522
10523
10524/**
10525 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10526 */
10527HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10528{
10529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10530
10531 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10532 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10533 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10534 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10535 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10536 {
10537 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10538 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10539 }
10540 AssertRCReturn(rc, rc);
10541 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10542
10543#ifdef VBOX_STRICT
10544 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10545 {
10546 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10547 && pMixedCtx->ecx != MSR_K6_EFER)
10548 {
10549 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10550 HMVMX_RETURN_UNEXPECTED_EXIT();
10551 }
10552# if HC_ARCH_BITS == 64
10553 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10554 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10555 {
10556 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10557 HMVMX_RETURN_UNEXPECTED_EXIT();
10558 }
10559# endif
10560 }
10561#endif
10562
10563 PVM pVM = pVCpu->CTX_SUFF(pVM);
10564 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10565 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10566 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10568 if (RT_LIKELY(rc == VINF_SUCCESS))
10569 {
10570 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10571 Assert(pVmxTransient->cbInstr == 2);
10572 }
10573 return rc;
10574}
10575
10576
10577/**
10578 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10579 */
10580HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10581{
10582 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10583 PVM pVM = pVCpu->CTX_SUFF(pVM);
10584 int rc = VINF_SUCCESS;
10585
10586 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10587 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10588 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10589 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10590 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10591 {
10592 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10593 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10594 }
10595 AssertRCReturn(rc, rc);
10596 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10597
10598 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10599 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10600 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10601
10602 if (RT_LIKELY(rc == VINF_SUCCESS))
10603 {
10604 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10605
10606 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10607 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10608 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10609 {
10610 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10611 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10612 EMInterpretWrmsr() changes it. */
10613 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10614 }
10615 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10616 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10617 else if (pMixedCtx->ecx == MSR_K6_EFER)
10618 {
10619 /*
10620 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10621 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10622 * the other bits as well, SCE and NXE. See @bugref{7368}.
10623 */
10624 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10625 }
10626
10627 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10628 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10629 {
10630 switch (pMixedCtx->ecx)
10631 {
10632 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10633 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10634 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10635 case MSR_K8_FS_BASE: /* no break */
10636 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10637 case MSR_K6_EFER: /* already handled above */ break;
10638 default:
10639 {
10640 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10641 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10642#if HC_ARCH_BITS == 64
10643 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10644 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10645#endif
10646 break;
10647 }
10648 }
10649 }
10650#ifdef VBOX_STRICT
10651 else
10652 {
10653 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10654 switch (pMixedCtx->ecx)
10655 {
10656 case MSR_IA32_SYSENTER_CS:
10657 case MSR_IA32_SYSENTER_EIP:
10658 case MSR_IA32_SYSENTER_ESP:
10659 case MSR_K8_FS_BASE:
10660 case MSR_K8_GS_BASE:
10661 {
10662 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10663 HMVMX_RETURN_UNEXPECTED_EXIT();
10664 }
10665
10666 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10667 default:
10668 {
10669 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10670 {
10671 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10672 if (pMixedCtx->ecx != MSR_K6_EFER)
10673 {
10674 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10675 pMixedCtx->ecx));
10676 HMVMX_RETURN_UNEXPECTED_EXIT();
10677 }
10678 }
10679
10680#if HC_ARCH_BITS == 64
10681 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10682 {
10683 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10684 HMVMX_RETURN_UNEXPECTED_EXIT();
10685 }
10686#endif
10687 break;
10688 }
10689 }
10690 }
10691#endif /* VBOX_STRICT */
10692 }
10693 return rc;
10694}
10695
10696
10697/**
10698 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10699 */
10700HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10701{
10702 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10703
10704 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10705 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10706 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10707 return VERR_EM_INTERPRETER;
10708 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10709 HMVMX_RETURN_UNEXPECTED_EXIT();
10710}
10711
10712
10713/**
10714 * VM-exit handler for when the TPR value is lowered below the specified
10715 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10716 */
10717HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10718{
10719 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10720 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10721
10722 /*
10723 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10724 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10725 * resume guest execution.
10726 */
10727 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10728 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10729 return VINF_SUCCESS;
10730}
10731
10732
10733/**
10734 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10735 * VM-exit.
10736 *
10737 * @retval VINF_SUCCESS when guest execution can continue.
10738 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10739 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10740 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10741 * recompiler.
10742 */
10743HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10744{
10745 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10746 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10747 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10748 AssertRCReturn(rc, rc);
10749
10750 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10751 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10752 PVM pVM = pVCpu->CTX_SUFF(pVM);
10753 switch (uAccessType)
10754 {
10755 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10756 {
10757#if 0
10758 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10759 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10760#else
10761 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10762 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10763 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10764#endif
10765 AssertRCReturn(rc, rc);
10766
10767 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10768 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10769 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10770 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10771
10772 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10773 {
10774 case 0: /* CR0 */
10775 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10776 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10777 break;
10778 case 2: /* CR2 */
10779 /* Nothing to do here, CR2 it's not part of the VMCS. */
10780 break;
10781 case 3: /* CR3 */
10782 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10783 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10784 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10785 break;
10786 case 4: /* CR4 */
10787 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10788 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10789 break;
10790 case 8: /* CR8 */
10791 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10792 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10793 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10794 break;
10795 default:
10796 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10797 break;
10798 }
10799
10800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10801 break;
10802 }
10803
10804 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10805 {
10806 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10807 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10808 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10809 AssertRCReturn(rc, rc);
10810 Assert( !pVM->hm.s.fNestedPaging
10811 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10812 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10813
10814 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10815 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10816 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10817
10818 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10819 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10820 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10821 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10822 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10823 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10824 break;
10825 }
10826
10827 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10828 {
10829 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10830 AssertRCReturn(rc, rc);
10831 rc = EMInterpretCLTS(pVM, pVCpu);
10832 AssertRCReturn(rc, rc);
10833 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10834 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10835 Log4(("CRX CLTS write rc=%d\n", rc));
10836 break;
10837 }
10838
10839 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10840 {
10841 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10842 AssertRCReturn(rc, rc);
10843 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10844 if (RT_LIKELY(rc == VINF_SUCCESS))
10845 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10847 Log4(("CRX LMSW write rc=%d\n", rc));
10848 break;
10849 }
10850
10851 default:
10852 {
10853 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10854 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10855 }
10856 }
10857
10858 /* Validate possible error codes. */
10859 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10860 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10861 if (RT_SUCCESS(rc))
10862 {
10863 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10864 AssertRCReturn(rc2, rc2);
10865 }
10866
10867 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10868 return rc;
10869}
10870
10871
10872/**
10873 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10874 * VM-exit.
10875 */
10876HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10877{
10878 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10879 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10880
10881 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10882 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10883 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10884 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10885 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10886 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10887 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10888 AssertRCReturn(rc2, rc2);
10889
10890 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10891 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10892 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10893 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10894 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10895 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10896 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10897 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10898
10899 /* I/O operation lookup arrays. */
10900 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10901 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10902
10903 VBOXSTRICTRC rcStrict;
10904 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10905 uint32_t const cbInstr = pVmxTransient->cbInstr;
10906 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10907 PVM pVM = pVCpu->CTX_SUFF(pVM);
10908 if (fIOString)
10909 {
10910#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10911 /*
10912 * INS/OUTS - I/O String instruction.
10913 *
10914 * Use instruction-information if available, otherwise fall back on
10915 * interpreting the instruction.
10916 */
10917 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10918 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10919 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10920 {
10921 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10922 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10923 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10924 AssertRCReturn(rc2, rc2);
10925 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10926 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10927 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10928 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10929 if (fIOWrite)
10930 {
10931 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10932 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10933 }
10934 else
10935 {
10936 /*
10937 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10938 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10939 * See Intel Instruction spec. for "INS".
10940 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10941 */
10942 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10943 }
10944 }
10945 else
10946 {
10947 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10948 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10949 AssertRCReturn(rc2, rc2);
10950 rcStrict = IEMExecOne(pVCpu);
10951 }
10952 /** @todo IEM needs to be setting these flags somehow. */
10953 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10954 fUpdateRipAlready = true;
10955#else
10956 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10957 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10958 if (RT_SUCCESS(rcStrict))
10959 {
10960 if (fIOWrite)
10961 {
10962 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10963 (DISCPUMODE)pDis->uAddrMode, cbValue);
10964 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10965 }
10966 else
10967 {
10968 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10969 (DISCPUMODE)pDis->uAddrMode, cbValue);
10970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10971 }
10972 }
10973 else
10974 {
10975 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10976 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10977 }
10978#endif
10979 }
10980 else
10981 {
10982 /*
10983 * IN/OUT - I/O instruction.
10984 */
10985 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10986 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
10987 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10988 if (fIOWrite)
10989 {
10990 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10991 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10992 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10993 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10994 }
10995 else
10996 {
10997 uint32_t u32Result = 0;
10998 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10999 if (IOM_SUCCESS(rcStrict))
11000 {
11001 /* Save result of I/O IN instr. in AL/AX/EAX. */
11002 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11003 }
11004 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11005 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11006 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11007 }
11008 }
11009
11010 if (IOM_SUCCESS(rcStrict))
11011 {
11012 if (!fUpdateRipAlready)
11013 {
11014 pMixedCtx->rip += cbInstr;
11015 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11016 }
11017
11018 /*
11019 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11020 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11021 */
11022 if (fIOString)
11023 {
11024 /** @todo Single-step for INS/OUTS with REP prefix? */
11025 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11026 }
11027 else if (fStepping)
11028 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11029
11030 /*
11031 * If any I/O breakpoints are armed, we need to check if one triggered
11032 * and take appropriate action.
11033 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11034 */
11035 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11036 AssertRCReturn(rc2, rc2);
11037
11038 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11039 * execution engines about whether hyper BPs and such are pending. */
11040 uint32_t const uDr7 = pMixedCtx->dr[7];
11041 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11042 && X86_DR7_ANY_RW_IO(uDr7)
11043 && (pMixedCtx->cr4 & X86_CR4_DE))
11044 || DBGFBpIsHwIoArmed(pVM)))
11045 {
11046 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11047
11048 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11049 VMMRZCallRing3Disable(pVCpu);
11050 HM_DISABLE_PREEMPT_IF_NEEDED();
11051
11052 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
11053
11054 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11055 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11056 {
11057 /* Raise #DB. */
11058 if (fIsGuestDbgActive)
11059 ASMSetDR6(pMixedCtx->dr[6]);
11060 if (pMixedCtx->dr[7] != uDr7)
11061 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11062
11063 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11064 }
11065 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11066 else if ( rcStrict2 != VINF_SUCCESS
11067 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11068 rcStrict = rcStrict2;
11069
11070 HM_RESTORE_PREEMPT_IF_NEEDED();
11071 VMMRZCallRing3Enable(pVCpu);
11072 }
11073 }
11074
11075#ifdef DEBUG
11076 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11077 Assert(!fIOWrite);
11078 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11079 Assert(fIOWrite);
11080 else
11081 {
11082 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11083 * statuses, that the VMM device and some others may return. See
11084 * IOM_SUCCESS() for guidance. */
11085 AssertMsg( RT_FAILURE(rcStrict)
11086 || rcStrict == VINF_SUCCESS
11087 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11088 || rcStrict == VINF_EM_DBG_BREAKPOINT
11089 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11090 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11091 }
11092#endif
11093
11094 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11095 return VBOXSTRICTRC_TODO(rcStrict);
11096}
11097
11098
11099/**
11100 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11101 * VM-exit.
11102 */
11103HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11104{
11105 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11106
11107 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11108 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11109 AssertRCReturn(rc, rc);
11110 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11111 {
11112 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11113 AssertRCReturn(rc, rc);
11114 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11115 {
11116 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11117
11118 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11119 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11120
11121 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11122 Assert(!pVCpu->hm.s.Event.fPending);
11123 pVCpu->hm.s.Event.fPending = true;
11124 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11125 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11126 AssertRCReturn(rc, rc);
11127 if (fErrorCodeValid)
11128 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11129 else
11130 pVCpu->hm.s.Event.u32ErrCode = 0;
11131 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11132 && uVector == X86_XCPT_PF)
11133 {
11134 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11135 }
11136
11137 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11138 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11139 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11140 }
11141 }
11142
11143 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11144 * emulation. */
11145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11146 return VERR_EM_INTERPRETER;
11147}
11148
11149
11150/**
11151 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11152 */
11153HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11154{
11155 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11156 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11157 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11158 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11159 AssertRCReturn(rc, rc);
11160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11161 return VINF_EM_DBG_STEPPED;
11162}
11163
11164
11165/**
11166 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11167 */
11168HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11169{
11170 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11171
11172 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11173 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11174 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11175 return VINF_SUCCESS;
11176 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11177 return rc;
11178
11179#if 0
11180 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11181 * just sync the whole thing. */
11182 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11183#else
11184 /* Aggressive state sync. for now. */
11185 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11186 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11187 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11188#endif
11189 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11190 AssertRCReturn(rc, rc);
11191
11192 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11193 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11194 switch (uAccessType)
11195 {
11196 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11197 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11198 {
11199 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11200 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11201 {
11202 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11203 }
11204
11205 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11206 GCPhys &= PAGE_BASE_GC_MASK;
11207 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11208 PVM pVM = pVCpu->CTX_SUFF(pVM);
11209 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11210 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11211
11212 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11213 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11214 CPUMCTX2CORE(pMixedCtx), GCPhys);
11215 rc = VBOXSTRICTRC_VAL(rc2);
11216 Log4(("ApicAccess rc=%d\n", rc));
11217 if ( rc == VINF_SUCCESS
11218 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11219 || rc == VERR_PAGE_NOT_PRESENT)
11220 {
11221 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11222 | HM_CHANGED_GUEST_RSP
11223 | HM_CHANGED_GUEST_RFLAGS
11224 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11225 rc = VINF_SUCCESS;
11226 }
11227 break;
11228 }
11229
11230 default:
11231 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11232 rc = VINF_EM_RAW_EMULATE_INSTR;
11233 break;
11234 }
11235
11236 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11237 if (rc != VINF_SUCCESS)
11238 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccessToR3);
11239 return rc;
11240}
11241
11242
11243/**
11244 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11245 * VM-exit.
11246 */
11247HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11248{
11249 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11250
11251 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11252 if (pVmxTransient->fWasGuestDebugStateActive)
11253 {
11254 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11255 HMVMX_RETURN_UNEXPECTED_EXIT();
11256 }
11257
11258 int rc = VERR_INTERNAL_ERROR_5;
11259 if ( !DBGFIsStepping(pVCpu)
11260 && !pVCpu->hm.s.fSingleInstruction
11261 && !pVmxTransient->fWasHyperDebugStateActive)
11262 {
11263 /* Don't intercept MOV DRx and #DB any more. */
11264 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11265 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11266 AssertRCReturn(rc, rc);
11267
11268 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11269 {
11270#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11271 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11272 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11273 AssertRCReturn(rc, rc);
11274#endif
11275 }
11276
11277 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11278 VMMRZCallRing3Disable(pVCpu);
11279 HM_DISABLE_PREEMPT_IF_NEEDED();
11280
11281 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11282 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11283 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11284
11285 HM_RESTORE_PREEMPT_IF_NEEDED();
11286 VMMRZCallRing3Enable(pVCpu);
11287
11288#ifdef VBOX_WITH_STATISTICS
11289 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11290 AssertRCReturn(rc, rc);
11291 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11292 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11293 else
11294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11295#endif
11296 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11297 return VINF_SUCCESS;
11298 }
11299
11300 /*
11301 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11302 * Update the segment registers and DR7 from the CPU.
11303 */
11304 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11305 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11306 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11307 AssertRCReturn(rc, rc);
11308 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11309
11310 PVM pVM = pVCpu->CTX_SUFF(pVM);
11311 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11312 {
11313 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11314 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11315 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11316 if (RT_SUCCESS(rc))
11317 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11319 }
11320 else
11321 {
11322 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11323 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11324 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11325 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11326 }
11327
11328 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11329 if (RT_SUCCESS(rc))
11330 {
11331 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11332 AssertRCReturn(rc2, rc2);
11333 }
11334 return rc;
11335}
11336
11337
11338/**
11339 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11340 * Conditional VM-exit.
11341 */
11342HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11343{
11344 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11345 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11346
11347 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11348 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11349 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11350 return VINF_SUCCESS;
11351 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11352 return rc;
11353
11354 RTGCPHYS GCPhys = 0;
11355 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11356
11357#if 0
11358 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11359#else
11360 /* Aggressive state sync. for now. */
11361 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11362 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11363 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11364#endif
11365 AssertRCReturn(rc, rc);
11366
11367 /*
11368 * If we succeed, resume guest execution.
11369 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11370 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11371 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11372 * weird case. See @bugref{6043}.
11373 */
11374 PVM pVM = pVCpu->CTX_SUFF(pVM);
11375 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11376 rc = VBOXSTRICTRC_VAL(rc2);
11377 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11378 if ( rc == VINF_SUCCESS
11379 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11380 || rc == VERR_PAGE_NOT_PRESENT)
11381 {
11382 /* Successfully handled MMIO operation. */
11383 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11384 | HM_CHANGED_GUEST_RSP
11385 | HM_CHANGED_GUEST_RFLAGS
11386 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11387 rc = VINF_SUCCESS;
11388 }
11389 return rc;
11390}
11391
11392
11393/**
11394 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11395 * VM-exit.
11396 */
11397HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11398{
11399 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11400 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11401
11402 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11403 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11404 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11405 return VINF_SUCCESS;
11406 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11407 return rc;
11408
11409 RTGCPHYS GCPhys = 0;
11410 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11411 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11412#if 0
11413 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11414#else
11415 /* Aggressive state sync. for now. */
11416 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11417 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11418 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11419#endif
11420 AssertRCReturn(rc, rc);
11421
11422 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11423 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11424
11425 RTGCUINT uErrorCode = 0;
11426 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11427 uErrorCode |= X86_TRAP_PF_ID;
11428 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11429 uErrorCode |= X86_TRAP_PF_RW;
11430 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11431 uErrorCode |= X86_TRAP_PF_P;
11432
11433 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11434
11435 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11436 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11437
11438 /* Handle the pagefault trap for the nested shadow table. */
11439 PVM pVM = pVCpu->CTX_SUFF(pVM);
11440 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11441 TRPMResetTrap(pVCpu);
11442
11443 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11444 if ( rc == VINF_SUCCESS
11445 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11446 || rc == VERR_PAGE_NOT_PRESENT)
11447 {
11448 /* Successfully synced our nested page tables. */
11449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11450 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11451 | HM_CHANGED_GUEST_RSP
11452 | HM_CHANGED_GUEST_RFLAGS);
11453 return VINF_SUCCESS;
11454 }
11455
11456 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11457 return rc;
11458}
11459
11460/** @} */
11461
11462/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11463/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11464/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11465
11466/** @name VM-exit exception handlers.
11467 * @{
11468 */
11469
11470/**
11471 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11472 */
11473static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11474{
11475 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11477
11478 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11479 AssertRCReturn(rc, rc);
11480
11481 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11482 {
11483 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11484 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11485
11486 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11487 * provides VM-exit instruction length. If this causes problem later,
11488 * disassemble the instruction like it's done on AMD-V. */
11489 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11490 AssertRCReturn(rc2, rc2);
11491 return rc;
11492 }
11493
11494 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11495 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11496 return rc;
11497}
11498
11499
11500/**
11501 * VM-exit exception handler for #BP (Breakpoint exception).
11502 */
11503static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11504{
11505 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11507
11508 /** @todo Try optimize this by not saving the entire guest state unless
11509 * really needed. */
11510 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11511 AssertRCReturn(rc, rc);
11512
11513 PVM pVM = pVCpu->CTX_SUFF(pVM);
11514 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11515 if (rc == VINF_EM_RAW_GUEST_TRAP)
11516 {
11517 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11518 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11519 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11520 AssertRCReturn(rc, rc);
11521
11522 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11523 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11524 }
11525
11526 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11527 return rc;
11528}
11529
11530
11531/**
11532 * VM-exit exception handler for #DB (Debug exception).
11533 */
11534static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535{
11536 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11537 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11538 Log6(("XcptDB\n"));
11539
11540 /*
11541 * Get the DR6-like values from the exit qualification and pass it to DBGF
11542 * for processing.
11543 */
11544 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11545 AssertRCReturn(rc, rc);
11546
11547 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11548 uint64_t uDR6 = X86_DR6_INIT_VAL;
11549 uDR6 |= ( pVmxTransient->uExitQualification
11550 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11551
11552 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11553 if (rc == VINF_EM_RAW_GUEST_TRAP)
11554 {
11555 /*
11556 * The exception was for the guest. Update DR6, DR7.GD and
11557 * IA32_DEBUGCTL.LBR before forwarding it.
11558 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11559 */
11560 VMMRZCallRing3Disable(pVCpu);
11561 HM_DISABLE_PREEMPT_IF_NEEDED();
11562
11563 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11564 pMixedCtx->dr[6] |= uDR6;
11565 if (CPUMIsGuestDebugStateActive(pVCpu))
11566 ASMSetDR6(pMixedCtx->dr[6]);
11567
11568 HM_RESTORE_PREEMPT_IF_NEEDED();
11569 VMMRZCallRing3Enable(pVCpu);
11570
11571 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11572 AssertRCReturn(rc, rc);
11573
11574 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11575 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11576
11577 /* Paranoia. */
11578 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11579 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11580
11581 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11582 AssertRCReturn(rc, rc);
11583
11584 /*
11585 * Raise #DB in the guest.
11586 *
11587 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11588 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11589 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11590 *
11591 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11592 */
11593 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11594 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11595 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11596 AssertRCReturn(rc, rc);
11597 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11598 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11599 return VINF_SUCCESS;
11600 }
11601
11602 /*
11603 * Not a guest trap, must be a hypervisor related debug event then.
11604 * Update DR6 in case someone is interested in it.
11605 */
11606 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11607 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11608 CPUMSetHyperDR6(pVCpu, uDR6);
11609
11610 return rc;
11611}
11612
11613
11614/**
11615 * VM-exit exception handler for #NM (Device-not-available exception: floating
11616 * point exception).
11617 */
11618static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11619{
11620 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11621
11622 /* We require CR0 and EFER. EFER is always up-to-date. */
11623 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11624 AssertRCReturn(rc, rc);
11625
11626 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11627 VMMRZCallRing3Disable(pVCpu);
11628 HM_DISABLE_PREEMPT_IF_NEEDED();
11629
11630 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11631 if (pVmxTransient->fWasGuestFPUStateActive)
11632 {
11633 rc = VINF_EM_RAW_GUEST_TRAP;
11634 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11635 }
11636 else
11637 {
11638#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11639 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11640#endif
11641 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11642 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11643 }
11644
11645 HM_RESTORE_PREEMPT_IF_NEEDED();
11646 VMMRZCallRing3Enable(pVCpu);
11647
11648 if (rc == VINF_SUCCESS)
11649 {
11650 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11651 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11653 pVCpu->hm.s.fUseGuestFpu = true;
11654 }
11655 else
11656 {
11657 /* Forward #NM to the guest. */
11658 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11659 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11660 AssertRCReturn(rc, rc);
11661 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11662 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11664 }
11665
11666 return VINF_SUCCESS;
11667}
11668
11669
11670/**
11671 * VM-exit exception handler for #GP (General-protection exception).
11672 *
11673 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11674 */
11675static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11676{
11677 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11679
11680 int rc = VERR_INTERNAL_ERROR_5;
11681 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11682 {
11683#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11684 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11685 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11686 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11687 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11688 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11689 AssertRCReturn(rc, rc);
11690 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11691 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11692 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11693 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11694 return rc;
11695#else
11696 /* We don't intercept #GP. */
11697 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11698 NOREF(pVmxTransient);
11699 return VERR_VMX_UNEXPECTED_EXCEPTION;
11700#endif
11701 }
11702
11703 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11704 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11705
11706 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11707 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11708 AssertRCReturn(rc, rc);
11709
11710 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11711 uint32_t cbOp = 0;
11712 PVM pVM = pVCpu->CTX_SUFF(pVM);
11713 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11714 if (RT_SUCCESS(rc))
11715 {
11716 rc = VINF_SUCCESS;
11717 Assert(cbOp == pDis->cbInstr);
11718 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11719 switch (pDis->pCurInstr->uOpcode)
11720 {
11721 case OP_CLI:
11722 {
11723 pMixedCtx->eflags.Bits.u1IF = 0;
11724 pMixedCtx->rip += pDis->cbInstr;
11725 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11726 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11727 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11728 break;
11729 }
11730
11731 case OP_STI:
11732 {
11733 pMixedCtx->eflags.Bits.u1IF = 1;
11734 pMixedCtx->rip += pDis->cbInstr;
11735 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11736 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11737 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11738 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11739 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11740 break;
11741 }
11742
11743 case OP_HLT:
11744 {
11745 rc = VINF_EM_HALT;
11746 pMixedCtx->rip += pDis->cbInstr;
11747 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11749 break;
11750 }
11751
11752 case OP_POPF:
11753 {
11754 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11755 uint32_t cbParm;
11756 uint32_t uMask;
11757 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11758 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11759 {
11760 cbParm = 4;
11761 uMask = 0xffffffff;
11762 }
11763 else
11764 {
11765 cbParm = 2;
11766 uMask = 0xffff;
11767 }
11768
11769 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11770 RTGCPTR GCPtrStack = 0;
11771 X86EFLAGS Eflags;
11772 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11773 &GCPtrStack);
11774 if (RT_SUCCESS(rc))
11775 {
11776 Assert(sizeof(Eflags.u32) >= cbParm);
11777 Eflags.u32 = 0;
11778 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11779 }
11780 if (RT_FAILURE(rc))
11781 {
11782 rc = VERR_EM_INTERPRETER;
11783 break;
11784 }
11785 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11786 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11787 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11788 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11789 pMixedCtx->esp += cbParm;
11790 pMixedCtx->esp &= uMask;
11791 pMixedCtx->rip += pDis->cbInstr;
11792 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11793 | HM_CHANGED_GUEST_RSP
11794 | HM_CHANGED_GUEST_RFLAGS);
11795 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11796 if (fStepping)
11797 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11798
11799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11800 break;
11801 }
11802
11803 case OP_PUSHF:
11804 {
11805 uint32_t cbParm;
11806 uint32_t uMask;
11807 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11808 {
11809 cbParm = 4;
11810 uMask = 0xffffffff;
11811 }
11812 else
11813 {
11814 cbParm = 2;
11815 uMask = 0xffff;
11816 }
11817
11818 /* Get the stack pointer & push the contents of eflags onto the stack. */
11819 RTGCPTR GCPtrStack = 0;
11820 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11821 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11822 if (RT_FAILURE(rc))
11823 {
11824 rc = VERR_EM_INTERPRETER;
11825 break;
11826 }
11827 X86EFLAGS Eflags = pMixedCtx->eflags;
11828 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11829 Eflags.Bits.u1RF = 0;
11830 Eflags.Bits.u1VM = 0;
11831
11832 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11833 if (RT_FAILURE(rc))
11834 {
11835 rc = VERR_EM_INTERPRETER;
11836 break;
11837 }
11838 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11839 pMixedCtx->esp -= cbParm;
11840 pMixedCtx->esp &= uMask;
11841 pMixedCtx->rip += pDis->cbInstr;
11842 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11843 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11844 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11845 break;
11846 }
11847
11848 case OP_IRET:
11849 {
11850 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11851 * instruction reference. */
11852 RTGCPTR GCPtrStack = 0;
11853 uint32_t uMask = 0xffff;
11854 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11855 uint16_t aIretFrame[3];
11856 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11857 {
11858 rc = VERR_EM_INTERPRETER;
11859 break;
11860 }
11861 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11862 &GCPtrStack);
11863 if (RT_SUCCESS(rc))
11864 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11865 if (RT_FAILURE(rc))
11866 {
11867 rc = VERR_EM_INTERPRETER;
11868 break;
11869 }
11870 pMixedCtx->eip = 0;
11871 pMixedCtx->ip = aIretFrame[0];
11872 pMixedCtx->cs.Sel = aIretFrame[1];
11873 pMixedCtx->cs.ValidSel = aIretFrame[1];
11874 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11875 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11876 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11877 pMixedCtx->sp += sizeof(aIretFrame);
11878 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11879 | HM_CHANGED_GUEST_SEGMENT_REGS
11880 | HM_CHANGED_GUEST_RSP
11881 | HM_CHANGED_GUEST_RFLAGS);
11882 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11883 if (fStepping)
11884 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11885 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11887 break;
11888 }
11889
11890 case OP_INT:
11891 {
11892 uint16_t uVector = pDis->Param1.uValue & 0xff;
11893 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11894 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11895 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11896 break;
11897 }
11898
11899 case OP_INTO:
11900 {
11901 if (pMixedCtx->eflags.Bits.u1OF)
11902 {
11903 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11904 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11905 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11906 }
11907 break;
11908 }
11909
11910 default:
11911 {
11912 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11913 EMCODETYPE_SUPERVISOR);
11914 rc = VBOXSTRICTRC_VAL(rc2);
11915 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11916 /** @todo We have to set pending-debug exceptions here when the guest is
11917 * single-stepping depending on the instruction that was interpreted. */
11918 Log4(("#GP rc=%Rrc\n", rc));
11919 break;
11920 }
11921 }
11922 }
11923 else
11924 rc = VERR_EM_INTERPRETER;
11925
11926 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11927 ("#GP Unexpected rc=%Rrc\n", rc));
11928 return rc;
11929}
11930
11931
11932#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11933/**
11934 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11935 * the exception reported in the VMX transient structure back into the VM.
11936 *
11937 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11938 * up-to-date.
11939 */
11940static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11941{
11942 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11943
11944 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11945 hmR0VmxCheckExitDueToEventDelivery(). */
11946 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11947 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11948 AssertRCReturn(rc, rc);
11949 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11950
11951#ifdef DEBUG_ramshankar
11952 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11953 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11954 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11955#endif
11956
11957 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11958 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11959 return VINF_SUCCESS;
11960}
11961#endif
11962
11963
11964/**
11965 * VM-exit exception handler for #PF (Page-fault exception).
11966 */
11967static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11968{
11969 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11970 PVM pVM = pVCpu->CTX_SUFF(pVM);
11971 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11972 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11973 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11974 AssertRCReturn(rc, rc);
11975
11976#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11977 if (pVM->hm.s.fNestedPaging)
11978 {
11979 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11980 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
11981 {
11982 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11983 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11984 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11985 }
11986 else
11987 {
11988 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11989 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11990 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11991 }
11992 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11993 return rc;
11994 }
11995#else
11996 Assert(!pVM->hm.s.fNestedPaging);
11997 NOREF(pVM);
11998#endif
11999
12000 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12001 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12002 if (pVmxTransient->fVectoringPF)
12003 {
12004 Assert(pVCpu->hm.s.Event.fPending);
12005 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12006 }
12007
12008 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12009 AssertRCReturn(rc, rc);
12010
12011 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12012 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12013
12014 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12015 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12016 (RTGCPTR)pVmxTransient->uExitQualification);
12017
12018 Log4(("#PF: rc=%Rrc\n", rc));
12019 if (rc == VINF_SUCCESS)
12020 {
12021 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12022 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12023 * memory? We don't update the whole state here... */
12024 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12025 | HM_CHANGED_GUEST_RSP
12026 | HM_CHANGED_GUEST_RFLAGS
12027 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12028 TRPMResetTrap(pVCpu);
12029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12030 return rc;
12031 }
12032 else if (rc == VINF_EM_RAW_GUEST_TRAP)
12033 {
12034 if (!pVmxTransient->fVectoringDoublePF)
12035 {
12036 /* It's a guest page fault and needs to be reflected to the guest. */
12037 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12038 TRPMResetTrap(pVCpu);
12039 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12040 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12041 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12042 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12043 }
12044 else
12045 {
12046 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12047 TRPMResetTrap(pVCpu);
12048 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12049 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12050 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12051 }
12052
12053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12054 return VINF_SUCCESS;
12055 }
12056
12057 TRPMResetTrap(pVCpu);
12058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12059 return rc;
12060}
12061
12062/** @} */
12063
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette