VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 52343

Last change on this file since 52343 was 52281, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: Fix strict assertion on older CPUs; check for secondary processor-based execution
controls before reading it off the VMCS.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 499.0 KB
Line 
1/* $Id: HMVMXR0.cpp 52281 2014-08-05 14:33:31Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns VBox status code.
323 * @param pVCpu Pointer to the VMCPU.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337
338/*******************************************************************************
339* Internal Functions *
340*******************************************************************************/
341static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
342static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
343static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
344 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
345#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
346static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
347#endif
348#ifndef HMVMX_USE_FUNCTION_TABLE
349DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
350# define HMVMX_EXIT_DECL static int
351#else
352# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
353#endif
354
355/** @name VM-exit handlers.
356 * @{
357 */
358static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
359static FNVMXEXITHANDLER hmR0VmxExitExtInt;
360static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
361static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
362static FNVMXEXITHANDLER hmR0VmxExitSipi;
363static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
364static FNVMXEXITHANDLER hmR0VmxExitSmi;
365static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
366static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
367static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
368static FNVMXEXITHANDLER hmR0VmxExitCpuid;
369static FNVMXEXITHANDLER hmR0VmxExitGetsec;
370static FNVMXEXITHANDLER hmR0VmxExitHlt;
371static FNVMXEXITHANDLER hmR0VmxExitInvd;
372static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
373static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
374static FNVMXEXITHANDLER hmR0VmxExitVmcall;
375static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
376static FNVMXEXITHANDLER hmR0VmxExitRsm;
377static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
378static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
379static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
380static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
381static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
382static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
383static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
384static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
385static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
386static FNVMXEXITHANDLER hmR0VmxExitMwait;
387static FNVMXEXITHANDLER hmR0VmxExitMtf;
388static FNVMXEXITHANDLER hmR0VmxExitMonitor;
389static FNVMXEXITHANDLER hmR0VmxExitPause;
390static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
391static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
392static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
395static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
396static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
397static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
398static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
399static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
400static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
401static FNVMXEXITHANDLER hmR0VmxExitRdrand;
402static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
403/** @} */
404
405static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
412static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413#endif
414static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
415
416/*******************************************************************************
417* Global Variables *
418*******************************************************************************/
419#ifdef HMVMX_USE_FUNCTION_TABLE
420
421/**
422 * VMX_EXIT dispatch table.
423 */
424static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
425{
426 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
427 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
428 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
429 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
430 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
431 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
432 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
433 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
434 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
435 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
436 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
437 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
438 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
439 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
440 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
441 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
442 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
443 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
444 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
445 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
446 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
447 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
448 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
449 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
450 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
451 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
452 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
453 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
454 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
455 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
456 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
457 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
458 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
459 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
460 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
461 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
462 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
463 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
464 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
465 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
466 /* 40 UNDEFINED */ hmR0VmxExitPause,
467 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
468 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
469 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
470 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
471 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
473 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
475 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
476 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
477 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
478 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
479 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
480 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
481 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
482 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
483 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
484 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
485 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
486};
487#endif /* HMVMX_USE_FUNCTION_TABLE */
488
489#ifdef VBOX_STRICT
490static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
491{
492 /* 0 */ "(Not Used)",
493 /* 1 */ "VMCALL executed in VMX root operation.",
494 /* 2 */ "VMCLEAR with invalid physical address.",
495 /* 3 */ "VMCLEAR with VMXON pointer.",
496 /* 4 */ "VMLAUNCH with non-clear VMCS.",
497 /* 5 */ "VMRESUME with non-launched VMCS.",
498 /* 6 */ "VMRESUME after VMXOFF",
499 /* 7 */ "VM entry with invalid control fields.",
500 /* 8 */ "VM entry with invalid host state fields.",
501 /* 9 */ "VMPTRLD with invalid physical address.",
502 /* 10 */ "VMPTRLD with VMXON pointer.",
503 /* 11 */ "VMPTRLD with incorrect revision identifier.",
504 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
505 /* 13 */ "VMWRITE to read-only VMCS component.",
506 /* 14 */ "(Not Used)",
507 /* 15 */ "VMXON executed in VMX root operation.",
508 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
509 /* 17 */ "VM entry with non-launched executing VMCS.",
510 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
511 /* 19 */ "VMCALL with non-clear VMCS.",
512 /* 20 */ "VMCALL with invalid VM-exit control fields.",
513 /* 21 */ "(Not Used)",
514 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
515 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
516 /* 24 */ "VMCALL with invalid SMM-monitor features.",
517 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
518 /* 26 */ "VM entry with events blocked by MOV SS.",
519 /* 27 */ "(Not Used)",
520 /* 28 */ "Invalid operand to INVEPT/INVVPID."
521};
522#endif /* VBOX_STRICT */
523
524
525
526/**
527 * Updates the VM's last error record. If there was a VMX instruction error,
528 * reads the error data from the VMCS and updates VCPU's last error record as
529 * well.
530 *
531 * @param pVM Pointer to the VM.
532 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
533 * VERR_VMX_UNABLE_TO_START_VM or
534 * VERR_VMX_INVALID_VMCS_FIELD).
535 * @param rc The error code.
536 */
537static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
538{
539 AssertPtr(pVM);
540 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
541 || rc == VERR_VMX_UNABLE_TO_START_VM)
542 {
543 AssertPtrReturnVoid(pVCpu);
544 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
545 }
546 pVM->hm.s.lLastError = rc;
547}
548
549
550/**
551 * Reads the VM-entry interruption-information field from the VMCS into the VMX
552 * transient structure.
553 *
554 * @returns VBox status code.
555 * @param pVmxTransient Pointer to the VMX transient structure.
556 *
557 * @remarks No-long-jump zone!!!
558 */
559DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
560{
561 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
562 AssertRCReturn(rc, rc);
563 return VINF_SUCCESS;
564}
565
566
567/**
568 * Reads the VM-entry exception error code field from the VMCS into
569 * the VMX transient structure.
570 *
571 * @returns VBox status code.
572 * @param pVmxTransient Pointer to the VMX transient structure.
573 *
574 * @remarks No-long-jump zone!!!
575 */
576DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
577{
578 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
579 AssertRCReturn(rc, rc);
580 return VINF_SUCCESS;
581}
582
583
584/**
585 * Reads the VM-entry exception error code field from the VMCS into
586 * the VMX transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600
601/**
602 * Reads the VM-exit interruption-information field from the VMCS into the VMX
603 * transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 */
608DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
611 {
612 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
613 AssertRCReturn(rc, rc);
614 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
615 }
616 return VINF_SUCCESS;
617}
618
619
620/**
621 * Reads the VM-exit interruption error code from the VMCS into the VMX
622 * transient structure.
623 *
624 * @returns VBox status code.
625 * @param pVmxTransient Pointer to the VMX transient structure.
626 */
627DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
630 {
631 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
632 AssertRCReturn(rc, rc);
633 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
634 }
635 return VINF_SUCCESS;
636}
637
638
639/**
640 * Reads the VM-exit instruction length field from the VMCS into the VMX
641 * transient structure.
642 *
643 * @returns VBox status code.
644 * @param pVCpu Pointer to the VMCPU.
645 * @param pVmxTransient Pointer to the VMX transient structure.
646 */
647DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
648{
649 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
650 {
651 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
652 AssertRCReturn(rc, rc);
653 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
654 }
655 return VINF_SUCCESS;
656}
657
658
659/**
660 * Reads the VM-exit instruction-information field from the VMCS into
661 * the VMX transient structure.
662 *
663 * @returns VBox status code.
664 * @param pVmxTransient Pointer to the VMX transient structure.
665 */
666DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
667{
668 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
669 {
670 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
671 AssertRCReturn(rc, rc);
672 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
673 }
674 return VINF_SUCCESS;
675}
676
677
678/**
679 * Reads the exit qualification from the VMCS into the VMX transient structure.
680 *
681 * @returns VBox status code.
682 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
683 * case).
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
689 {
690 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the IDT-vectoring information field from the VMCS into the VMX
700 * transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 *
705 * @remarks No-long-jump zone!!!
706 */
707DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the IDT-vectoring error code from the VMCS into the VMX
721 * transient structure.
722 *
723 * @returns VBox status code.
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
729 {
730 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Enters VMX root mode operation on the current CPU.
740 *
741 * @returns VBox status code.
742 * @param pVM Pointer to the VM (optional, can be NULL, after
743 * a resume).
744 * @param HCPhysCpuPage Physical address of the VMXON region.
745 * @param pvCpuPage Pointer to the VMXON region.
746 */
747static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
748{
749 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
750 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
751 Assert(pvCpuPage);
752 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
753
754 if (pVM)
755 {
756 /* Write the VMCS revision dword to the VMXON region. */
757 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
758 }
759
760 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
761 RTCCUINTREG uEflags = ASMIntDisableFlags();
762
763 /* Enable the VMX bit in CR4 if necessary. */
764 RTCCUINTREG uCr4 = ASMGetCR4();
765 if (!(uCr4 & X86_CR4_VMXE))
766 ASMSetCR4(uCr4 | X86_CR4_VMXE);
767
768 /* Enter VMX root mode. */
769 int rc = VMXEnable(HCPhysCpuPage);
770 if (RT_FAILURE(rc))
771 ASMSetCR4(uCr4);
772
773 /* Restore interrupts. */
774 ASMSetFlags(uEflags);
775 return rc;
776}
777
778
779/**
780 * Exits VMX root mode operation on the current CPU.
781 *
782 * @returns VBox status code.
783 */
784static int hmR0VmxLeaveRootMode(void)
785{
786 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
787
788 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
789 RTCCUINTREG uEflags = ASMIntDisableFlags();
790
791 /* If we're for some reason not in VMX root mode, then don't leave it. */
792 RTCCUINTREG uHostCR4 = ASMGetCR4();
793
794 int rc;
795 if (uHostCR4 & X86_CR4_VMXE)
796 {
797 /* Exit VMX root mode and clear the VMX bit in CR4. */
798 VMXDisable();
799 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
800 rc = VINF_SUCCESS;
801 }
802 else
803 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
804
805 /* Restore interrupts. */
806 ASMSetFlags(uEflags);
807 return rc;
808}
809
810
811/**
812 * Allocates and maps one physically contiguous page. The allocated page is
813 * zero'd out. (Used by various VT-x structures).
814 *
815 * @returns IPRT status code.
816 * @param pMemObj Pointer to the ring-0 memory object.
817 * @param ppVirt Where to store the virtual address of the
818 * allocation.
819 * @param pPhys Where to store the physical address of the
820 * allocation.
821 */
822DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
823{
824 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
825 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
826 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
827
828 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
829 if (RT_FAILURE(rc))
830 return rc;
831 *ppVirt = RTR0MemObjAddress(*pMemObj);
832 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
833 ASMMemZero32(*ppVirt, PAGE_SIZE);
834 return VINF_SUCCESS;
835}
836
837
838/**
839 * Frees and unmaps an allocated physical page.
840 *
841 * @param pMemObj Pointer to the ring-0 memory object.
842 * @param ppVirt Where to re-initialize the virtual address of
843 * allocation as 0.
844 * @param pHCPhys Where to re-initialize the physical address of the
845 * allocation as 0.
846 */
847DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
848{
849 AssertPtr(pMemObj);
850 AssertPtr(ppVirt);
851 AssertPtr(pHCPhys);
852 if (*pMemObj != NIL_RTR0MEMOBJ)
853 {
854 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
855 AssertRC(rc);
856 *pMemObj = NIL_RTR0MEMOBJ;
857 *ppVirt = 0;
858 *pHCPhys = 0;
859 }
860}
861
862
863/**
864 * Worker function to free VT-x related structures.
865 *
866 * @returns IPRT status code.
867 * @param pVM Pointer to the VM.
868 */
869static void hmR0VmxStructsFree(PVM pVM)
870{
871 for (VMCPUID i = 0; i < pVM->cCpus; i++)
872 {
873 PVMCPU pVCpu = &pVM->aCpus[i];
874 AssertPtr(pVCpu);
875
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
878
879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
881
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
884 }
885
886 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
887#ifdef VBOX_WITH_CRASHDUMP_MAGIC
888 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
889#endif
890}
891
892
893/**
894 * Worker function to allocate VT-x related VM structures.
895 *
896 * @returns IPRT status code.
897 * @param pVM Pointer to the VM.
898 */
899static int hmR0VmxStructsAlloc(PVM pVM)
900{
901 /*
902 * Initialize members up-front so we can cleanup properly on allocation failure.
903 */
904#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVM->hm.s.vmx.HCPhys##a_Name = 0;
908
909#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
910 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
911 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
912 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
913
914#ifdef VBOX_WITH_CRASHDUMP_MAGIC
915 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
916#endif
917 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
918
919 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
925 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
926 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
927 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
928 }
929#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
930#undef VMXLOCAL_INIT_VM_MEMOBJ
931
932 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
933 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
934 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
935 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
936
937 /*
938 * Allocate all the VT-x structures.
939 */
940 int rc = VINF_SUCCESS;
941#ifdef VBOX_WITH_CRASHDUMP_MAGIC
942 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
943 if (RT_FAILURE(rc))
944 goto cleanup;
945 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
946 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
947#endif
948
949 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
950 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
951 {
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
953 &pVM->hm.s.vmx.HCPhysApicAccess);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 }
957
958 /*
959 * Initialize per-VCPU VT-x structures.
960 */
961 for (VMCPUID i = 0; i < pVM->cCpus; i++)
962 {
963 PVMCPU pVCpu = &pVM->aCpus[i];
964 AssertPtr(pVCpu);
965
966 /* Allocate the VM control structure (VMCS). */
967 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
968 if (RT_FAILURE(rc))
969 goto cleanup;
970
971 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
972 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
973 {
974 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
975 &pVCpu->hm.s.vmx.HCPhysVirtApic);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978 }
979
980 /*
981 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
982 * transparent accesses of specific MSRs.
983 *
984 * If the condition for enabling MSR bitmaps changes here, don't forget to
985 * update HMIsMsrBitmapsAvailable().
986 */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
988 {
989 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
990 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
994 }
995
996 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
997 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000
1001 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1002 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1003 if (RT_FAILURE(rc))
1004 goto cleanup;
1005 }
1006
1007 return VINF_SUCCESS;
1008
1009cleanup:
1010 hmR0VmxStructsFree(pVM);
1011 return rc;
1012}
1013
1014
1015/**
1016 * Does global VT-x initialization (called during module initialization).
1017 *
1018 * @returns VBox status code.
1019 */
1020VMMR0DECL(int) VMXR0GlobalInit(void)
1021{
1022#ifdef HMVMX_USE_FUNCTION_TABLE
1023 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1024# ifdef VBOX_STRICT
1025 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1026 Assert(g_apfnVMExitHandlers[i]);
1027# endif
1028#endif
1029 return VINF_SUCCESS;
1030}
1031
1032
1033/**
1034 * Does global VT-x termination (called during module termination).
1035 */
1036VMMR0DECL(void) VMXR0GlobalTerm()
1037{
1038 /* Nothing to do currently. */
1039}
1040
1041
1042/**
1043 * Sets up and activates VT-x on the current CPU.
1044 *
1045 * @returns VBox status code.
1046 * @param pCpu Pointer to the global CPU info struct.
1047 * @param pVM Pointer to the VM (can be NULL after a host resume
1048 * operation).
1049 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1050 * fEnabledByHost is true).
1051 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1052 * @a fEnabledByHost is true).
1053 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1054 * enable VT-x on the host.
1055 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1056 */
1057VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1058 void *pvMsrs)
1059{
1060 Assert(pCpu);
1061 Assert(pvMsrs);
1062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1063
1064 /* Enable VT-x if it's not already enabled by the host. */
1065 if (!fEnabledByHost)
1066 {
1067 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1068 if (RT_FAILURE(rc))
1069 return rc;
1070 }
1071
1072 /*
1073 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1074 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1075 */
1076 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1077 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1078 {
1079 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1080 pCpu->fFlushAsidBeforeUse = false;
1081 }
1082 else
1083 pCpu->fFlushAsidBeforeUse = true;
1084
1085 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1086 ++pCpu->cTlbFlushes;
1087
1088 return VINF_SUCCESS;
1089}
1090
1091
1092/**
1093 * Deactivates VT-x on the current CPU.
1094 *
1095 * @returns VBox status code.
1096 * @param pCpu Pointer to the global CPU info struct.
1097 * @param pvCpuPage Pointer to the VMXON region.
1098 * @param HCPhysCpuPage Physical address of the VMXON region.
1099 *
1100 * @remarks This function should never be called when SUPR0EnableVTx() or
1101 * similar was used to enable VT-x on the host.
1102 */
1103VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1104{
1105 NOREF(pCpu);
1106 NOREF(pvCpuPage);
1107 NOREF(HCPhysCpuPage);
1108
1109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1110 return hmR0VmxLeaveRootMode();
1111}
1112
1113
1114/**
1115 * Sets the permission bits for the specified MSR in the MSR bitmap.
1116 *
1117 * @param pVCpu Pointer to the VMCPU.
1118 * @param uMSR The MSR value.
1119 * @param enmRead Whether reading this MSR causes a VM-exit.
1120 * @param enmWrite Whether writing this MSR causes a VM-exit.
1121 */
1122static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1123{
1124 int32_t iBit;
1125 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1126
1127 /*
1128 * Layout:
1129 * 0x000 - 0x3ff - Low MSR read bits
1130 * 0x400 - 0x7ff - High MSR read bits
1131 * 0x800 - 0xbff - Low MSR write bits
1132 * 0xc00 - 0xfff - High MSR write bits
1133 */
1134 if (uMsr <= 0x00001FFF)
1135 iBit = uMsr;
1136 else if ( uMsr >= 0xC0000000
1137 && uMsr <= 0xC0001FFF)
1138 {
1139 iBit = (uMsr - 0xC0000000);
1140 pbMsrBitmap += 0x400;
1141 }
1142 else
1143 {
1144 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1145 return;
1146 }
1147
1148 Assert(iBit <= 0x1fff);
1149 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1150 ASMBitSet(pbMsrBitmap, iBit);
1151 else
1152 ASMBitClear(pbMsrBitmap, iBit);
1153
1154 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1155 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1156 else
1157 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1158}
1159
1160
1161#ifdef VBOX_STRICT
1162/**
1163 * Gets the permission bits for the specified MSR in the MSR bitmap.
1164 *
1165 * @returns VBox status code.
1166 * @retval VINF_SUCCESS if the specified MSR is found.
1167 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1168 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1169 *
1170 * @param pVCpu Pointer to the VMCPU.
1171 * @param uMsr The MSR.
1172 * @param penmRead Where to store the read permissions.
1173 * @param penmWrite Where to store the write permissions.
1174 */
1175static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1176{
1177 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1178 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1179 int32_t iBit;
1180 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1181
1182 /* See hmR0VmxSetMsrPermission() for the layout. */
1183 if (uMsr <= 0x00001FFF)
1184 iBit = uMsr;
1185 else if ( uMsr >= 0xC0000000
1186 && uMsr <= 0xC0001FFF)
1187 {
1188 iBit = (uMsr - 0xC0000000);
1189 pbMsrBitmap += 0x400;
1190 }
1191 else
1192 {
1193 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1194 return VERR_NOT_SUPPORTED;
1195 }
1196
1197 Assert(iBit <= 0x1fff);
1198 if (ASMBitTest(pbMsrBitmap, iBit))
1199 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1200 else
1201 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1202
1203 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1204 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1205 else
1206 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1207 return VINF_SUCCESS;
1208}
1209#endif /* VBOX_STRICT */
1210
1211
1212/**
1213 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1214 * area.
1215 *
1216 * @returns VBox status code.
1217 * @param pVCpu Pointer to the VMCPU.
1218 * @param cMsrs The number of MSRs.
1219 */
1220DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1221{
1222 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1223 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1224 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1225 {
1226 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1227 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1229 }
1230
1231 /* Update number of guest MSRs to load/store across the world-switch. */
1232 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1233 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1234
1235 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update the VCPU's copy of the MSR count. */
1239 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1240
1241 return VINF_SUCCESS;
1242}
1243
1244
1245/**
1246 * Adds a new (or updates the value of an existing) guest/host MSR
1247 * pair to be swapped during the world-switch as part of the
1248 * auto-load/store MSR area in the VMCS.
1249 *
1250 * @returns true if the MSR was added -and- its value was updated, false
1251 * otherwise.
1252 * @param pVCpu Pointer to the VMCPU.
1253 * @param uMsr The MSR.
1254 * @param uGuestMsr Value of the guest MSR.
1255 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1256 * necessary.
1257 */
1258static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cMsrs; i++)
1264 {
1265 if (pGuestMsr->u32Msr == uMsr)
1266 break;
1267 pGuestMsr++;
1268 }
1269
1270 bool fAdded = false;
1271 if (i == cMsrs)
1272 {
1273 ++cMsrs;
1274 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1275 AssertRC(rc);
1276
1277 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1278 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1279 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1280
1281 fAdded = true;
1282 }
1283
1284 /* Update the MSR values in the auto-load/store MSR area. */
1285 pGuestMsr->u32Msr = uMsr;
1286 pGuestMsr->u64Value = uGuestMsrValue;
1287
1288 /* Create/update the MSR slot in the host MSR area. */
1289 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1290 pHostMsr += i;
1291 pHostMsr->u32Msr = uMsr;
1292
1293 /*
1294 * Update the host MSR only when requested by the caller AND when we're
1295 * adding it to the auto-load/store area. Otherwise, it would have been
1296 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1297 */
1298 bool fUpdatedMsrValue = false;
1299 if ( fAdded
1300 && fUpdateHostMsr)
1301 {
1302 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1303 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1304 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1305 fUpdatedMsrValue = true;
1306 }
1307
1308 return fUpdatedMsrValue;
1309}
1310
1311
1312/**
1313 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1314 * auto-load/store MSR area in the VMCS.
1315 *
1316 * @returns VBox status code.
1317 * @param pVCpu Pointer to the VMCPU.
1318 * @param uMsr The MSR.
1319 */
1320static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1321{
1322 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1323 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1324 for (uint32_t i = 0; i < cMsrs; i++)
1325 {
1326 /* Find the MSR. */
1327 if (pGuestMsr->u32Msr == uMsr)
1328 {
1329 /* If it's the last MSR, simply reduce the count. */
1330 if (i == cMsrs - 1)
1331 {
1332 --cMsrs;
1333 break;
1334 }
1335
1336 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1337 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1338 pLastGuestMsr += cMsrs - 1;
1339 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1340 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1341
1342 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 pLastHostMsr += cMsrs - 1;
1345 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1346 pHostMsr->u64Value = pLastHostMsr->u64Value;
1347 --cMsrs;
1348 break;
1349 }
1350 pGuestMsr++;
1351 }
1352
1353 /* Update the VMCS if the count changed (meaning the MSR was found). */
1354 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1355 {
1356 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1357 AssertRCReturn(rc, rc);
1358
1359 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1360 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1361 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1362
1363 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1364 return VINF_SUCCESS;
1365 }
1366
1367 return VERR_NOT_FOUND;
1368}
1369
1370
1371/**
1372 * Checks if the specified guest MSR is part of the auto-load/store area in
1373 * the VMCS.
1374 *
1375 * @returns true if found, false otherwise.
1376 * @param pVCpu Pointer to the VMCPU.
1377 * @param uMsr The MSR to find.
1378 */
1379static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1380{
1381 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1383
1384 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1385 {
1386 if (pGuestMsr->u32Msr == uMsr)
1387 return true;
1388 }
1389 return false;
1390}
1391
1392
1393/**
1394 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1395 *
1396 * @param pVCpu Pointer to the VMCPU.
1397 *
1398 * @remarks No-long-jump zone!!!
1399 */
1400static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1401{
1402 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1403 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1405 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1406
1407 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1408 {
1409 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1410
1411 /*
1412 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1413 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1414 */
1415 if (pHostMsr->u32Msr == MSR_K6_EFER)
1416 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1417 else
1418 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1419 }
1420
1421 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1422}
1423
1424
1425#if HC_ARCH_BITS == 64
1426/**
1427 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1428 * perform lazy restoration of the host MSRs while leaving VT-x.
1429 *
1430 * @param pVCpu Pointer to the VMCPU.
1431 *
1432 * @remarks No-long-jump zone!!!
1433 */
1434static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1435{
1436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1437
1438 /*
1439 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1440 */
1441 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1442 {
1443 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1444 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1445 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1446 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1447 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1448 }
1449}
1450
1451
1452/**
1453 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1454 * lazily while leaving VT-x.
1455 *
1456 * @returns true if it does, false otherwise.
1457 * @param pVCpu Pointer to the VMCPU.
1458 * @param uMsr The MSR to check.
1459 */
1460static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1461{
1462 NOREF(pVCpu);
1463 switch (uMsr)
1464 {
1465 case MSR_K8_LSTAR:
1466 case MSR_K6_STAR:
1467 case MSR_K8_SF_MASK:
1468 case MSR_K8_KERNEL_GS_BASE:
1469 return true;
1470 }
1471 return false;
1472}
1473
1474
1475/**
1476 * Saves a set of guest MSRs back into the guest-CPU context.
1477 *
1478 * @param pVCpu Pointer to the VMCPU.
1479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1480 * out-of-sync. Make sure to update the required fields
1481 * before using them.
1482 *
1483 * @remarks No-long-jump zone!!!
1484 */
1485static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1486{
1487 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1488 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1489
1490 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1491 {
1492 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1493 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1494 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1495 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1496 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1497 }
1498}
1499
1500
1501/**
1502 * Loads a set of guests MSRs to allow read/passthru to the guest.
1503 *
1504 * The name of this function is slightly confusing. This function does NOT
1505 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1506 * common prefix for functions dealing with "lazy restoration" of the shared
1507 * MSRs.
1508 *
1509 * @param pVCpu Pointer to the VMCPU.
1510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1511 * out-of-sync. Make sure to update the required fields
1512 * before using them.
1513 *
1514 * @remarks No-long-jump zone!!!
1515 */
1516static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1517{
1518 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1520
1521#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1522 do { \
1523 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1524 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1525 else \
1526 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1527 } while (0)
1528
1529 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1530 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1531 {
1532 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1536 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1537 }
1538 else
1539 {
1540 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1541 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1542 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1543 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1544 }
1545
1546#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1547}
1548
1549
1550/**
1551 * Performs lazy restoration of the set of host MSRs if they were previously
1552 * loaded with guest MSR values.
1553 *
1554 * @param pVCpu Pointer to the VMCPU.
1555 *
1556 * @remarks No-long-jump zone!!!
1557 * @remarks The guest MSRs should have been saved back into the guest-CPU
1558 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1559 */
1560static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1561{
1562 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1563 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1564
1565 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1566 {
1567 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1568 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1569 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1570 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1571 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1572 }
1573 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1574}
1575#endif /* HC_ARCH_BITS == 64 */
1576
1577
1578/**
1579 * Verifies that our cached values of the VMCS controls are all
1580 * consistent with what's actually present in the VMCS.
1581 *
1582 * @returns VBox status code.
1583 * @param pVCpu Pointer to the VMCPU.
1584 */
1585static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1586{
1587 uint32_t u32Val;
1588 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1589 AssertRCReturn(rc, rc);
1590 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1591 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1592
1593 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1594 AssertRCReturn(rc, rc);
1595 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1596 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1597
1598 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1599 AssertRCReturn(rc, rc);
1600 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1601 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1602
1603 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1604 AssertRCReturn(rc, rc);
1605 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1606 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1607
1608 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1609 {
1610 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1611 AssertRCReturn(rc, rc);
1612 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1613 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1614 }
1615
1616 return VINF_SUCCESS;
1617}
1618
1619
1620#ifdef VBOX_STRICT
1621/**
1622 * Verifies that our cached host EFER value has not changed
1623 * since we cached it.
1624 *
1625 * @param pVCpu Pointer to the VMCPU.
1626 */
1627static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1628{
1629 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1630
1631 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1632 {
1633 uint64_t u64Val;
1634 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1635 AssertRC(rc);
1636
1637 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1638 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1639 }
1640}
1641
1642
1643/**
1644 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1645 * VMCS are correct.
1646 *
1647 * @param pVCpu Pointer to the VMCPU.
1648 */
1649static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1650{
1651 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1652
1653 /* Verify MSR counts in the VMCS are what we think it should be. */
1654 uint32_t cMsrs;
1655 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1656 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1657
1658 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1659 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1662 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1663
1664 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1665 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1666 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1667 {
1668 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1669 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1670 pGuestMsr->u32Msr, cMsrs));
1671
1672 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1673 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1674 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1675
1676 /* Verify that the permissions are as expected in the MSR bitmap. */
1677 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1678 {
1679 VMXMSREXITREAD enmRead;
1680 VMXMSREXITWRITE enmWrite;
1681 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1682 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1683 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1684 {
1685 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1686 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1687 }
1688 else
1689 {
1690 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1691 pGuestMsr->u32Msr, cMsrs));
1692 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1693 pGuestMsr->u32Msr, cMsrs));
1694 }
1695 }
1696 }
1697}
1698#endif /* VBOX_STRICT */
1699
1700
1701/**
1702 * Flushes the TLB using EPT.
1703 *
1704 * @returns VBox status code.
1705 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1706 * enmFlush).
1707 * @param enmFlush Type of flush.
1708 *
1709 * @remarks Caller is responsible for making sure this function is called only
1710 * when NestedPaging is supported and providing @a enmFlush that is
1711 * supported by the CPU.
1712 * @remarks Can be called with interrupts disabled.
1713 */
1714static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1715{
1716 uint64_t au64Descriptor[2];
1717 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1718 au64Descriptor[0] = 0;
1719 else
1720 {
1721 Assert(pVCpu);
1722 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1723 }
1724 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1725
1726 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1727 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1728 rc));
1729 if ( RT_SUCCESS(rc)
1730 && pVCpu)
1731 {
1732 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1733 }
1734}
1735
1736
1737/**
1738 * Flushes the TLB using VPID.
1739 *
1740 * @returns VBox status code.
1741 * @param pVM Pointer to the VM.
1742 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1743 * enmFlush).
1744 * @param enmFlush Type of flush.
1745 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1746 * on @a enmFlush).
1747 *
1748 * @remarks Can be called with interrupts disabled.
1749 */
1750static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1751{
1752 NOREF(pVM);
1753 AssertPtr(pVM);
1754 Assert(pVM->hm.s.vmx.fVpid);
1755
1756 uint64_t au64Descriptor[2];
1757 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1758 {
1759 au64Descriptor[0] = 0;
1760 au64Descriptor[1] = 0;
1761 }
1762 else
1763 {
1764 AssertPtr(pVCpu);
1765 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1766 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1767 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1768 au64Descriptor[1] = GCPtr;
1769 }
1770
1771 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1772 AssertMsg(rc == VINF_SUCCESS,
1773 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1774 if ( RT_SUCCESS(rc)
1775 && pVCpu)
1776 {
1777 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1778 }
1779}
1780
1781
1782/**
1783 * Invalidates a guest page by guest virtual address. Only relevant for
1784 * EPT/VPID, otherwise there is nothing really to invalidate.
1785 *
1786 * @returns VBox status code.
1787 * @param pVM Pointer to the VM.
1788 * @param pVCpu Pointer to the VMCPU.
1789 * @param GCVirt Guest virtual address of the page to invalidate.
1790 */
1791VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1792{
1793 AssertPtr(pVM);
1794 AssertPtr(pVCpu);
1795 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1796
1797 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1798 if (!fFlushPending)
1799 {
1800 /*
1801 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1802 * See @bugref{6043} and @bugref{6177}.
1803 *
1804 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1805 * function maybe called in a loop with individual addresses.
1806 */
1807 if (pVM->hm.s.vmx.fVpid)
1808 {
1809 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1810 {
1811 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1812 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1813 }
1814 else
1815 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1816 }
1817 else if (pVM->hm.s.fNestedPaging)
1818 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1819 }
1820
1821 return VINF_SUCCESS;
1822}
1823
1824
1825/**
1826 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1827 * otherwise there is nothing really to invalidate.
1828 *
1829 * @returns VBox status code.
1830 * @param pVM Pointer to the VM.
1831 * @param pVCpu Pointer to the VMCPU.
1832 * @param GCPhys Guest physical address of the page to invalidate.
1833 */
1834VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1835{
1836 NOREF(pVM); NOREF(GCPhys);
1837 LogFlowFunc(("%RGp\n", GCPhys));
1838
1839 /*
1840 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1841 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1842 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1843 */
1844 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1845 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1846 return VINF_SUCCESS;
1847}
1848
1849
1850/**
1851 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1852 * case where neither EPT nor VPID is supported by the CPU.
1853 *
1854 * @param pVM Pointer to the VM.
1855 * @param pVCpu Pointer to the VMCPU.
1856 * @param pCpu Pointer to the global HM struct.
1857 *
1858 * @remarks Called with interrupts disabled.
1859 */
1860static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1861{
1862 AssertPtr(pVCpu);
1863 AssertPtr(pCpu);
1864 NOREF(pVM);
1865
1866 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1867
1868 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1869#if 0
1870 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1871 pVCpu->hm.s.TlbShootdown.cPages = 0;
1872#endif
1873
1874 Assert(pCpu->idCpu != NIL_RTCPUID);
1875 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1876 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1877 pVCpu->hm.s.fForceTLBFlush = false;
1878 return;
1879}
1880
1881
1882/**
1883 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1884 *
1885 * @param pVM Pointer to the VM.
1886 * @param pVCpu Pointer to the VMCPU.
1887 * @param pCpu Pointer to the global HM CPU struct.
1888 * @remarks All references to "ASID" in this function pertains to "VPID" in
1889 * Intel's nomenclature. The reason is, to avoid confusion in compare
1890 * statements since the host-CPU copies are named "ASID".
1891 *
1892 * @remarks Called with interrupts disabled.
1893 */
1894static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1895{
1896#ifdef VBOX_WITH_STATISTICS
1897 bool fTlbFlushed = false;
1898# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1899# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1900 if (!fTlbFlushed) \
1901 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1902 } while (0)
1903#else
1904# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1905# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1906#endif
1907
1908 AssertPtr(pVM);
1909 AssertPtr(pCpu);
1910 AssertPtr(pVCpu);
1911 Assert(pCpu->idCpu != NIL_RTCPUID);
1912
1913 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1914 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1915 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1916
1917 /*
1918 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1919 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1920 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1921 */
1922 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1923 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1924 {
1925 ++pCpu->uCurrentAsid;
1926 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1927 {
1928 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1929 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1930 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1931 }
1932
1933 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1934 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1935 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1936
1937 /*
1938 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1939 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1940 */
1941 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1942 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1943 HMVMX_SET_TAGGED_TLB_FLUSHED();
1944 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1945 }
1946
1947 /* Check for explicit TLB shootdowns. */
1948 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1949 {
1950 /*
1951 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1952 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1953 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1954 * but not guest-physical mappings.
1955 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1956 */
1957 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1958 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1959 HMVMX_SET_TAGGED_TLB_FLUSHED();
1960 }
1961
1962 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1963 * where it is commented out. Support individual entry flushing
1964 * someday. */
1965#if 0
1966 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1967 {
1968 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1969
1970 /*
1971 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1972 * as supported by the CPU.
1973 */
1974 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1975 {
1976 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1977 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1978 }
1979 else
1980 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1981
1982 HMVMX_SET_TAGGED_TLB_FLUSHED();
1983 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1984 pVCpu->hm.s.TlbShootdown.cPages = 0;
1985 }
1986#endif
1987
1988 pVCpu->hm.s.fForceTLBFlush = false;
1989
1990 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1991
1992 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1993 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1994 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1995 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1996 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1997 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1998 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1999 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2000 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2001
2002 /* Update VMCS with the VPID. */
2003 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2004 AssertRC(rc);
2005
2006#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2007}
2008
2009
2010/**
2011 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2012 *
2013 * @returns VBox status code.
2014 * @param pVM Pointer to the VM.
2015 * @param pVCpu Pointer to the VMCPU.
2016 * @param pCpu Pointer to the global HM CPU struct.
2017 *
2018 * @remarks Called with interrupts disabled.
2019 */
2020static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2021{
2022 AssertPtr(pVM);
2023 AssertPtr(pVCpu);
2024 AssertPtr(pCpu);
2025 Assert(pCpu->idCpu != NIL_RTCPUID);
2026 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2027 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2028
2029 /*
2030 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2031 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2032 */
2033 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2034 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2035 {
2036 pVCpu->hm.s.fForceTLBFlush = true;
2037 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2038 }
2039
2040 /* Check for explicit TLB shootdown flushes. */
2041 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2042 {
2043 pVCpu->hm.s.fForceTLBFlush = true;
2044 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2045 }
2046
2047 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2048 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2049
2050 if (pVCpu->hm.s.fForceTLBFlush)
2051 {
2052 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2053 pVCpu->hm.s.fForceTLBFlush = false;
2054 }
2055 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2056 * where it is commented out. Support individual entry flushing
2057 * someday. */
2058#if 0
2059 else
2060 {
2061 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2062 {
2063 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2065 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2066 }
2067 else
2068 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2069
2070 pVCpu->hm.s.TlbShootdown.cPages = 0;
2071 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2072 }
2073#endif
2074}
2075
2076
2077/**
2078 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2079 *
2080 * @returns VBox status code.
2081 * @param pVM Pointer to the VM.
2082 * @param pVCpu Pointer to the VMCPU.
2083 * @param pCpu Pointer to the global HM CPU struct.
2084 *
2085 * @remarks Called with interrupts disabled.
2086 */
2087static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2088{
2089 AssertPtr(pVM);
2090 AssertPtr(pVCpu);
2091 AssertPtr(pCpu);
2092 Assert(pCpu->idCpu != NIL_RTCPUID);
2093 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2094 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2095
2096 /*
2097 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2098 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2099 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2100 */
2101 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2102 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2103 {
2104 pVCpu->hm.s.fForceTLBFlush = true;
2105 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2106 }
2107
2108 /* Check for explicit TLB shootdown flushes. */
2109 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2110 {
2111 /*
2112 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2113 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2114 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2115 */
2116 pVCpu->hm.s.fForceTLBFlush = true;
2117 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2118 }
2119
2120 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2121 if (pVCpu->hm.s.fForceTLBFlush)
2122 {
2123 ++pCpu->uCurrentAsid;
2124 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2125 {
2126 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2127 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2128 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2129 }
2130
2131 pVCpu->hm.s.fForceTLBFlush = false;
2132 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2133 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2134 if (pCpu->fFlushAsidBeforeUse)
2135 {
2136 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2137 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2138 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2139 {
2140 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2141 pCpu->fFlushAsidBeforeUse = false;
2142 }
2143 else
2144 {
2145 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2146 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2147 }
2148 }
2149 }
2150 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2151 * where it is commented out. Support individual entry flushing
2152 * someday. */
2153#if 0
2154 else
2155 {
2156 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2157 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2158 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2159 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2160
2161 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2162 {
2163 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2164 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2165 {
2166 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2167 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2168 }
2169 else
2170 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2171
2172 pVCpu->hm.s.TlbShootdown.cPages = 0;
2173 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2174 }
2175 else
2176 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2177 }
2178#endif
2179
2180 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2181 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2182 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2183 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2184 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2185 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2186 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2187
2188 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2189 AssertRC(rc);
2190}
2191
2192
2193/**
2194 * Flushes the guest TLB entry based on CPU capabilities.
2195 *
2196 * @param pVCpu Pointer to the VMCPU.
2197 * @param pCpu Pointer to the global HM CPU struct.
2198 */
2199DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2200{
2201#ifdef HMVMX_ALWAYS_FLUSH_TLB
2202 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2203#endif
2204 PVM pVM = pVCpu->CTX_SUFF(pVM);
2205 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2206 {
2207 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2208 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2209 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2210 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2211 default:
2212 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2213 break;
2214 }
2215
2216 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2217 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2218
2219 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2220}
2221
2222
2223/**
2224 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2225 * TLB entries from the host TLB before VM-entry.
2226 *
2227 * @returns VBox status code.
2228 * @param pVM Pointer to the VM.
2229 */
2230static int hmR0VmxSetupTaggedTlb(PVM pVM)
2231{
2232 /*
2233 * Determine optimal flush type for Nested Paging.
2234 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2235 * guest execution (see hmR3InitFinalizeR0()).
2236 */
2237 if (pVM->hm.s.fNestedPaging)
2238 {
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2240 {
2241 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2242 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2243 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2244 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2245 else
2246 {
2247 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2248 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2249 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2250 }
2251
2252 /* Make sure the write-back cacheable memory type for EPT is supported. */
2253 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2254 {
2255 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2256 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2257 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2258 }
2259 }
2260 else
2261 {
2262 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2263 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2264 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2265 }
2266 }
2267
2268 /*
2269 * Determine optimal flush type for VPID.
2270 */
2271 if (pVM->hm.s.vmx.fVpid)
2272 {
2273 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2274 {
2275 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2276 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2277 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2278 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2279 else
2280 {
2281 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2282 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2283 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2284 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2285 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2286 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2287 pVM->hm.s.vmx.fVpid = false;
2288 }
2289 }
2290 else
2291 {
2292 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2293 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2294 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2295 pVM->hm.s.vmx.fVpid = false;
2296 }
2297 }
2298
2299 /*
2300 * Setup the handler for flushing tagged-TLBs.
2301 */
2302 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2303 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2304 else if (pVM->hm.s.fNestedPaging)
2305 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2306 else if (pVM->hm.s.vmx.fVpid)
2307 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2308 else
2309 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2310 return VINF_SUCCESS;
2311}
2312
2313
2314/**
2315 * Sets up pin-based VM-execution controls in the VMCS.
2316 *
2317 * @returns VBox status code.
2318 * @param pVM Pointer to the VM.
2319 * @param pVCpu Pointer to the VMCPU.
2320 */
2321static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2322{
2323 AssertPtr(pVM);
2324 AssertPtr(pVCpu);
2325
2326 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2327 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2328
2329 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2330 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2331
2332 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2333 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2334
2335 /* Enable the VMX preemption timer. */
2336 if (pVM->hm.s.vmx.fUsePreemptTimer)
2337 {
2338 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2339 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2340 }
2341
2342 if ((val & zap) != val)
2343 {
2344 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2345 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2346 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2347 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2348 }
2349
2350 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2351 AssertRCReturn(rc, rc);
2352
2353 /* Update VCPU with the currently set pin-based VM-execution controls. */
2354 pVCpu->hm.s.vmx.u32PinCtls = val;
2355 return rc;
2356}
2357
2358
2359/**
2360 * Sets up processor-based VM-execution controls in the VMCS.
2361 *
2362 * @returns VBox status code.
2363 * @param pVM Pointer to the VM.
2364 * @param pVMCPU Pointer to the VMCPU.
2365 */
2366static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2367{
2368 AssertPtr(pVM);
2369 AssertPtr(pVCpu);
2370
2371 int rc = VERR_INTERNAL_ERROR_5;
2372 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2373 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2374
2375 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2378 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2382
2383 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2384 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2385 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2386 {
2387 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2388 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2389 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2390 }
2391
2392 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2393 if (!pVM->hm.s.fNestedPaging)
2394 {
2395 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2396 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2397 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2398 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2399 }
2400
2401 /* Use TPR shadowing if supported by the CPU. */
2402 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2403 {
2404 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2405 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2406 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2407 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2408 AssertRCReturn(rc, rc);
2409
2410 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2411 /* CR8 writes cause a VM-exit based on TPR threshold. */
2412 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2413 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2414 }
2415 else
2416 {
2417 /*
2418 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2419 * Set this control only for 64-bit guests.
2420 */
2421 if (pVM->hm.s.fAllow64BitGuests)
2422 {
2423 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2424 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2425 }
2426 }
2427
2428 /* Use MSR-bitmaps if supported by the CPU. */
2429 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2430 {
2431 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2432
2433 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2434 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2435 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2436 AssertRCReturn(rc, rc);
2437
2438 /*
2439 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2440 * automatically using dedicated fields in the VMCS.
2441 */
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447
2448#if HC_ARCH_BITS == 64
2449 /*
2450 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2451 */
2452 if (pVM->hm.s.fAllow64BitGuests)
2453 {
2454 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2456 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 }
2459#endif
2460 }
2461
2462 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2463 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2464 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2465
2466 if ((val & zap) != val)
2467 {
2468 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2469 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2470 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2471 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2472 }
2473
2474 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2475 AssertRCReturn(rc, rc);
2476
2477 /* Update VCPU with the currently set processor-based VM-execution controls. */
2478 pVCpu->hm.s.vmx.u32ProcCtls = val;
2479
2480 /*
2481 * Secondary processor-based VM-execution controls.
2482 */
2483 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2484 {
2485 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2486 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2487
2488 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2489 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2490
2491 if (pVM->hm.s.fNestedPaging)
2492 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2493 else
2494 {
2495 /*
2496 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2497 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2498 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2499 */
2500 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2502 }
2503
2504 if (pVM->hm.s.vmx.fVpid)
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2506
2507 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2509
2510 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2511 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2512 * done dynamically. */
2513 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2514 {
2515 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2516 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2517 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2518 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2519 AssertRCReturn(rc, rc);
2520 }
2521
2522 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2523 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2524
2525 if ((val & zap) != val)
2526 {
2527 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2528 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2529 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2530 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2531 }
2532
2533 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2534 AssertRCReturn(rc, rc);
2535
2536 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2537 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2538 }
2539 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2542 "available\n"));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 return VINF_SUCCESS;
2548}
2549
2550
2551/**
2552 * Sets up miscellaneous (everything other than Pin & Processor-based
2553 * VM-execution) control fields in the VMCS.
2554 *
2555 * @returns VBox status code.
2556 * @param pVM Pointer to the VM.
2557 * @param pVCpu Pointer to the VMCPU.
2558 */
2559static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2560{
2561 NOREF(pVM);
2562 AssertPtr(pVM);
2563 AssertPtr(pVCpu);
2564
2565 int rc = VERR_GENERAL_FAILURE;
2566
2567 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2568#if 0
2569 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2570 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2571 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2572
2573 /*
2574 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2575 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2576 * We thus use the exception bitmap to control it rather than use both.
2577 */
2578 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2580
2581 /** @todo Explore possibility of using IO-bitmaps. */
2582 /* All IO & IOIO instructions cause VM-exits. */
2583 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2584 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2585
2586 /* Initialize the MSR-bitmap area. */
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2590#endif
2591
2592 /* Setup MSR auto-load/store area. */
2593 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2595 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 AssertRCReturn(rc, rc);
2597 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599
2600 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 AssertRCReturn(rc, rc);
2604
2605 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2607 AssertRCReturn(rc, rc);
2608
2609 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2610#if 0
2611 /* Setup debug controls */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2613 AssertRCReturn(rc, rc);
2614 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 return rc;
2619}
2620
2621
2622/**
2623 * Sets up the initial exception bitmap in the VMCS based on static conditions
2624 * (i.e. conditions that cannot ever change after starting the VM).
2625 *
2626 * @returns VBox status code.
2627 * @param pVM Pointer to the VM.
2628 * @param pVCpu Pointer to the VMCPU.
2629 */
2630static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2631{
2632 AssertPtr(pVM);
2633 AssertPtr(pVCpu);
2634
2635 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2636
2637 uint32_t u32XcptBitmap = 0;
2638
2639 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2640 if (!pVM->hm.s.fNestedPaging)
2641 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2642
2643 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2644 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2645 AssertRCReturn(rc, rc);
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial guest-state mask. The guest-state mask is consulted
2652 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2653 * for the nested virtualization case (as it would cause a VM-exit).
2654 *
2655 * @param pVCpu Pointer to the VMCPU.
2656 */
2657static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2658{
2659 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2660 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x initialization.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM Pointer to the VM.
2670 */
2671VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675 int rc = hmR0VmxStructsAlloc(pVM);
2676 if (RT_FAILURE(rc))
2677 {
2678 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2679 return rc;
2680 }
2681
2682 return VINF_SUCCESS;
2683}
2684
2685
2686/**
2687 * Does per-VM VT-x termination.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM Pointer to the VM.
2691 */
2692VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2693{
2694 LogFlowFunc(("pVM=%p\n", pVM));
2695
2696#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2697 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2698 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2699#endif
2700 hmR0VmxStructsFree(pVM);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Sets up the VM for execution under VT-x.
2707 * This function is only called once per-VM during initialization.
2708 *
2709 * @returns VBox status code.
2710 * @param pVM Pointer to the VM.
2711 */
2712VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2713{
2714 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2716
2717 LogFlowFunc(("pVM=%p\n", pVM));
2718
2719 /*
2720 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2721 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2722 */
2723 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2724 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2725 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2726 || !pVM->hm.s.vmx.pRealModeTSS))
2727 {
2728 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2729 return VERR_INTERNAL_ERROR;
2730 }
2731
2732#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2733 /*
2734 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2735 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2736 */
2737 if ( pVM->hm.s.fAllow64BitGuests
2738 && !HMVMX_IS_64BIT_HOST_MODE())
2739 {
2740 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2741 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2742 }
2743#endif
2744
2745 /* Initialize these always, see hmR3InitFinalizeR0().*/
2746 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2747 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2748
2749 /* Setup the tagged-TLB flush handlers. */
2750 int rc = hmR0VmxSetupTaggedTlb(pVM);
2751 if (RT_FAILURE(rc))
2752 {
2753 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2754 return rc;
2755 }
2756
2757 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2758 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2759#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2760 if ( HMVMX_IS_64BIT_HOST_MODE()
2761 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2763 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2764 {
2765 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2766 }
2767#endif
2768
2769 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2770 {
2771 PVMCPU pVCpu = &pVM->aCpus[i];
2772 AssertPtr(pVCpu);
2773 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2774
2775 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2776 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2777
2778 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2779 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2780 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2781
2782 /* Set revision dword at the beginning of the VMCS structure. */
2783 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2784
2785 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2786 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2788 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2789
2790 /* Load this VMCS as the current VMCS. */
2791 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2801 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2802
2803 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2805 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2809 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2810
2811 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2812 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2813 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2814
2815#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2816 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819#endif
2820
2821 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2822 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2827
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2829 }
2830
2831 return VINF_SUCCESS;
2832}
2833
2834
2835/**
2836 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2837 * the VMCS.
2838 *
2839 * @returns VBox status code.
2840 * @param pVM Pointer to the VM.
2841 * @param pVCpu Pointer to the VMCPU.
2842 */
2843DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2844{
2845 NOREF(pVM); NOREF(pVCpu);
2846
2847 RTCCUINTREG uReg = ASMGetCR0();
2848 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2849 AssertRCReturn(rc, rc);
2850
2851#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2852 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2853 if (HMVMX_IS_64BIT_HOST_MODE())
2854 {
2855 uint64_t uRegCR3 = HMR0Get64bitCR3();
2856 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2857 }
2858 else
2859#endif
2860 {
2861 uReg = ASMGetCR3();
2862 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2863 }
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR4();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2868 AssertRCReturn(rc, rc);
2869 return rc;
2870}
2871
2872
2873#if HC_ARCH_BITS == 64
2874/**
2875 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2876 * requirements. See hmR0VmxSaveHostSegmentRegs().
2877 */
2878# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2879 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2880 { \
2881 bool fValidSelector = true; \
2882 if ((selValue) & X86_SEL_LDT) \
2883 { \
2884 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2885 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2886 } \
2887 if (fValidSelector) \
2888 { \
2889 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2890 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2891 } \
2892 (selValue) = 0; \
2893 }
2894#endif
2895
2896
2897/**
2898 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2899 * the host-state area in the VMCS.
2900 *
2901 * @returns VBox status code.
2902 * @param pVM Pointer to the VM.
2903 * @param pVCpu Pointer to the VMCPU.
2904 */
2905DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2906{
2907 int rc = VERR_INTERNAL_ERROR_5;
2908
2909#if HC_ARCH_BITS == 64
2910 /*
2911 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2912 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2913 */
2914 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2915 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2916#endif
2917
2918 /*
2919 * Host DS, ES, FS and GS segment registers.
2920 */
2921#if HC_ARCH_BITS == 64
2922 RTSEL uSelDS = ASMGetDS();
2923 RTSEL uSelES = ASMGetES();
2924 RTSEL uSelFS = ASMGetFS();
2925 RTSEL uSelGS = ASMGetGS();
2926#else
2927 RTSEL uSelDS = 0;
2928 RTSEL uSelES = 0;
2929 RTSEL uSelFS = 0;
2930 RTSEL uSelGS = 0;
2931#endif
2932
2933 /* Recalculate which host-state bits need to be manually restored. */
2934 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2935
2936 /*
2937 * Host CS and SS segment registers.
2938 */
2939#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2940 RTSEL uSelCS;
2941 RTSEL uSelSS;
2942 if (HMVMX_IS_64BIT_HOST_MODE())
2943 {
2944 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2945 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2946 }
2947 else
2948 {
2949 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2950 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2951 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2952 }
2953#else
2954 RTSEL uSelCS = ASMGetCS();
2955 RTSEL uSelSS = ASMGetSS();
2956#endif
2957
2958 /*
2959 * Host TR segment register.
2960 */
2961 RTSEL uSelTR = ASMGetTR();
2962
2963#if HC_ARCH_BITS == 64
2964 /*
2965 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2966 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2967 */
2968 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2969 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2970 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2971 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2972# undef VMXLOCAL_ADJUST_HOST_SEG
2973#endif
2974
2975 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2976 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2977 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2978 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2979 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2980 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2981 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2982 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2983 Assert(uSelCS);
2984 Assert(uSelTR);
2985
2986 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2987#if 0
2988 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2989 Assert(uSelSS != 0);
2990#endif
2991
2992 /* Write these host selector fields into the host-state area in the VMCS. */
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2994 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2995#if HC_ARCH_BITS == 64
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2999 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
3000#endif
3001 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3002
3003 /*
3004 * Host GDTR and IDTR.
3005 */
3006 RTGDTR Gdtr;
3007 RT_ZERO(Gdtr);
3008#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3009 if (HMVMX_IS_64BIT_HOST_MODE())
3010 {
3011 X86XDTR64 Gdtr64;
3012 X86XDTR64 Idtr64;
3013 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3015 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3016
3017 Gdtr.cbGdt = Gdtr64.cb;
3018 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3019 }
3020 else
3021#endif
3022 {
3023 RTIDTR Idtr;
3024 ASMGetGDTR(&Gdtr);
3025 ASMGetIDTR(&Idtr);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3027 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3028
3029#if HC_ARCH_BITS == 64
3030 /*
3031 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3032 * maximum limit (0xffff) on every VM-exit.
3033 */
3034 if (Gdtr.cbGdt != 0xffff)
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3037 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3039 }
3040
3041 /*
3042 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3043 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3044 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3045 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3046 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3047 * hosts where we are pretty sure it won't cause trouble.
3048 */
3049# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3050 if (Idtr.cbIdt < 0x0fff)
3051# else
3052 if (Idtr.cbIdt != 0xffff)
3053# endif
3054 {
3055 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3056 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3057 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3058 }
3059#endif
3060 }
3061
3062 /*
3063 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3064 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3065 */
3066 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3067 {
3068 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3069 return VERR_VMX_INVALID_HOST_STATE;
3070 }
3071
3072 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3073#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3074 if (HMVMX_IS_64BIT_HOST_MODE())
3075 {
3076 /* We need the 64-bit TR base for hybrid darwin. */
3077 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3078 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3079 }
3080 else
3081#endif
3082 {
3083 uintptr_t uTRBase;
3084#if HC_ARCH_BITS == 64
3085 uTRBase = X86DESC64_BASE(pDesc);
3086
3087 /*
3088 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3089 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3090 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3091 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3092 *
3093 * [1] See Intel spec. 3.5 "System Descriptor Types".
3094 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3095 */
3096 Assert(pDesc->System.u4Type == 11);
3097 if ( pDesc->System.u16LimitLow != 0x67
3098 || pDesc->System.u4LimitHigh)
3099 {
3100 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3101 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3102 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3103 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3104 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3105
3106 /* Store the GDTR here as we need it while restoring TR. */
3107 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3108 }
3109#else
3110 uTRBase = X86DESC_BASE(pDesc);
3111#endif
3112 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3113 }
3114 AssertRCReturn(rc, rc);
3115
3116 /*
3117 * Host FS base and GS base.
3118 */
3119#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3120 if (HMVMX_IS_64BIT_HOST_MODE())
3121 {
3122 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3123 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3124 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3125 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3126
3127# if HC_ARCH_BITS == 64
3128 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3129 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3130 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3131 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3132 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3133# endif
3134 }
3135#endif
3136 return rc;
3137}
3138
3139
3140/**
3141 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3142 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3143 * the host after every successful VM-exit.
3144 *
3145 * @returns VBox status code.
3146 * @param pVM Pointer to the VM.
3147 * @param pVCpu Pointer to the VMCPU.
3148 *
3149 * @remarks No-long-jump zone!!!
3150 */
3151DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3152{
3153 NOREF(pVM);
3154
3155 AssertPtr(pVCpu);
3156 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3157
3158 int rc = VINF_SUCCESS;
3159#if HC_ARCH_BITS == 64
3160 if (pVM->hm.s.fAllow64BitGuests)
3161 hmR0VmxLazySaveHostMsrs(pVCpu);
3162#endif
3163
3164 /*
3165 * Host Sysenter MSRs.
3166 */
3167 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3168 AssertRCReturn(rc, rc);
3169#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3170 if (HMVMX_IS_64BIT_HOST_MODE())
3171 {
3172 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3173 AssertRCReturn(rc, rc);
3174 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3175 }
3176 else
3177 {
3178 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3179 AssertRCReturn(rc, rc);
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3181 }
3182#elif HC_ARCH_BITS == 32
3183 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3184 AssertRCReturn(rc, rc);
3185 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3186#else
3187 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3188 AssertRCReturn(rc, rc);
3189 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3190#endif
3191 AssertRCReturn(rc, rc);
3192
3193 /*
3194 * Host EFER MSR.
3195 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3196 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3197 */
3198 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3199 {
3200 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3201 AssertRCReturn(rc, rc);
3202 }
3203
3204 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3205 * hmR0VmxLoadGuestExitCtls() !! */
3206
3207 return rc;
3208}
3209
3210
3211/**
3212 * Figures out if we need to swap the EFER MSR which is
3213 * particularly expensive.
3214 *
3215 * We check all relevant bits. For now, that's everything
3216 * besides LMA/LME, as these two bits are handled by VM-entry,
3217 * see hmR0VmxLoadGuestExitCtls() and
3218 * hmR0VMxLoadGuestEntryCtls().
3219 *
3220 * @returns true if we need to load guest EFER, false otherwise.
3221 * @param pVCpu Pointer to the VMCPU.
3222 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3223 * out-of-sync. Make sure to update the required fields
3224 * before using them.
3225 *
3226 * @remarks Requires EFER, CR4.
3227 * @remarks No-long-jump zone!!!
3228 */
3229static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3230{
3231#ifdef HMVMX_ALWAYS_SWAP_EFER
3232 return true;
3233#endif
3234
3235#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3236 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3237 if (CPUMIsGuestInLongMode(pVCpu))
3238 return false;
3239#endif
3240
3241 PVM pVM = pVCpu->CTX_SUFF(pVM);
3242 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3243 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3244
3245 /*
3246 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3247 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3248 */
3249 if ( CPUMIsGuestInLongMode(pVCpu)
3250 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3251 {
3252 return true;
3253 }
3254
3255 /*
3256 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3257 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3258 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3259 */
3260 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3261 && (pMixedCtx->cr0 & X86_CR0_PG)
3262 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3263 {
3264 /* Assert that host is PAE capable. */
3265 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3266 return true;
3267 }
3268
3269 /** @todo Check the latest Intel spec. for any other bits,
3270 * like SMEP/SMAP? */
3271 return false;
3272}
3273
3274
3275/**
3276 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3277 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3278 * controls".
3279 *
3280 * @returns VBox status code.
3281 * @param pVCpu Pointer to the VMCPU.
3282 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3283 * out-of-sync. Make sure to update the required fields
3284 * before using them.
3285 *
3286 * @remarks Requires EFER.
3287 * @remarks No-long-jump zone!!!
3288 */
3289DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3290{
3291 int rc = VINF_SUCCESS;
3292 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3293 {
3294 PVM pVM = pVCpu->CTX_SUFF(pVM);
3295 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3296 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3297
3298 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3299 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3300
3301 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3302 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3303 {
3304 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3305 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3306 }
3307 else
3308 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3309
3310 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3311 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3312 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3313 {
3314 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3315 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3316 }
3317
3318 /*
3319 * The following should -not- be set (since we're not in SMM mode):
3320 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3321 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3322 */
3323
3324 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3325 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3326
3327 if ((val & zap) != val)
3328 {
3329 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3330 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3331 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3332 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3333 }
3334
3335 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3336 AssertRCReturn(rc, rc);
3337
3338 /* Update VCPU with the currently set VM-exit controls. */
3339 pVCpu->hm.s.vmx.u32EntryCtls = val;
3340 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3341 }
3342 return rc;
3343}
3344
3345
3346/**
3347 * Sets up the VM-exit controls in the VMCS.
3348 *
3349 * @returns VBox status code.
3350 * @param pVM Pointer to the VM.
3351 * @param pVCpu Pointer to the VMCPU.
3352 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3353 * out-of-sync. Make sure to update the required fields
3354 * before using them.
3355 *
3356 * @remarks Requires EFER.
3357 */
3358DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3359{
3360 NOREF(pMixedCtx);
3361
3362 int rc = VINF_SUCCESS;
3363 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3364 {
3365 PVM pVM = pVCpu->CTX_SUFF(pVM);
3366 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3367 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3368
3369 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3370 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3371
3372 /*
3373 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3374 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3375 */
3376#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3377 if (HMVMX_IS_64BIT_HOST_MODE())
3378 {
3379 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3380 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3381 }
3382 else
3383 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3384#else
3385 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3386 {
3387 /* The switcher returns to long mode, EFER is managed by the switcher. */
3388 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3389 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3390 }
3391 else
3392 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3393#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3394
3395 /* If the newer VMCS fields for managing EFER exists, use it. */
3396 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3397 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3398 {
3399 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3400 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3401 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3402 }
3403
3404 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3405 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3406
3407 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3408 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3409 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3410
3411 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3412 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3413
3414 if ((val & zap) != val)
3415 {
3416 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3417 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3418 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3419 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3420 }
3421
3422 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3423 AssertRCReturn(rc, rc);
3424
3425 /* Update VCPU with the currently set VM-exit controls. */
3426 pVCpu->hm.s.vmx.u32ExitCtls = val;
3427 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3428 }
3429 return rc;
3430}
3431
3432
3433/**
3434 * Loads the guest APIC and related state.
3435 *
3436 * @returns VBox status code.
3437 * @param pVM Pointer to the VM.
3438 * @param pVCpu Pointer to the VMCPU.
3439 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3440 * out-of-sync. Make sure to update the required fields
3441 * before using them.
3442 */
3443DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3444{
3445 NOREF(pMixedCtx);
3446
3447 int rc = VINF_SUCCESS;
3448 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3449 {
3450 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3451 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3452 {
3453 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3454
3455 bool fPendingIntr = false;
3456 uint8_t u8Tpr = 0;
3457 uint8_t u8PendingIntr = 0;
3458 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3459 AssertRCReturn(rc, rc);
3460
3461 /*
3462 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3463 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3464 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3465 * the interrupt when we VM-exit for other reasons.
3466 */
3467 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3468 uint32_t u32TprThreshold = 0;
3469 if (fPendingIntr)
3470 {
3471 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3472 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3473 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3474 if (u8PendingPriority <= u8TprPriority)
3475 u32TprThreshold = u8PendingPriority;
3476 else
3477 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3478 }
3479 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3480
3481 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3482 AssertRCReturn(rc, rc);
3483 }
3484
3485 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3486 }
3487 return rc;
3488}
3489
3490
3491/**
3492 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3493 *
3494 * @returns Guest's interruptibility-state.
3495 * @param pVCpu Pointer to the VMCPU.
3496 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3497 * out-of-sync. Make sure to update the required fields
3498 * before using them.
3499 *
3500 * @remarks No-long-jump zone!!!
3501 */
3502DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3503{
3504 /*
3505 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3506 */
3507 uint32_t uIntrState = 0;
3508 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3509 {
3510 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3511 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3512 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3513 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3514 {
3515 if (pMixedCtx->eflags.Bits.u1IF)
3516 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3517 else
3518 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3519 }
3520 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3521 }
3522
3523 /*
3524 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3525 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3526 * setting this would block host-NMIs and IRET will not clear the blocking.
3527 *
3528 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3529 */
3530 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3531 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3532 {
3533 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3534 }
3535
3536 return uIntrState;
3537}
3538
3539
3540/**
3541 * Loads the guest's interruptibility-state into the guest-state area in the
3542 * VMCS.
3543 *
3544 * @returns VBox status code.
3545 * @param pVCpu Pointer to the VMCPU.
3546 * @param uIntrState The interruptibility-state to set.
3547 */
3548static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3549{
3550 NOREF(pVCpu);
3551 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3552 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3553 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3554 AssertRCReturn(rc, rc);
3555 return rc;
3556}
3557
3558
3559/**
3560 * Loads the guest's RIP into the guest-state area in the VMCS.
3561 *
3562 * @returns VBox status code.
3563 * @param pVCpu Pointer to the VMCPU.
3564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565 * out-of-sync. Make sure to update the required fields
3566 * before using them.
3567 *
3568 * @remarks No-long-jump zone!!!
3569 */
3570static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3571{
3572 int rc = VINF_SUCCESS;
3573 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3574 {
3575 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3576 AssertRCReturn(rc, rc);
3577
3578 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3579 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3580 HMCPU_CF_VALUE(pVCpu)));
3581 }
3582 return rc;
3583}
3584
3585
3586/**
3587 * Loads the guest's RSP into the guest-state area in the VMCS.
3588 *
3589 * @returns VBox status code.
3590 * @param pVCpu Pointer to the VMCPU.
3591 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3592 * out-of-sync. Make sure to update the required fields
3593 * before using them.
3594 *
3595 * @remarks No-long-jump zone!!!
3596 */
3597static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3598{
3599 int rc = VINF_SUCCESS;
3600 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3601 {
3602 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3603 AssertRCReturn(rc, rc);
3604
3605 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3606 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3607 }
3608 return rc;
3609}
3610
3611
3612/**
3613 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3614 *
3615 * @returns VBox status code.
3616 * @param pVCpu Pointer to the VMCPU.
3617 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618 * out-of-sync. Make sure to update the required fields
3619 * before using them.
3620 *
3621 * @remarks No-long-jump zone!!!
3622 */
3623static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3624{
3625 int rc = VINF_SUCCESS;
3626 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3627 {
3628 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3629 Let us assert it as such and use 32-bit VMWRITE. */
3630 Assert(!(pMixedCtx->rflags.u64 >> 32));
3631 X86EFLAGS Eflags = pMixedCtx->eflags;
3632 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3633 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3634
3635 /*
3636 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3637 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3638 */
3639 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3640 {
3641 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3642 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3643 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3644 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3645 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3646 }
3647
3648 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3649 AssertRCReturn(rc, rc);
3650
3651 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3652 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3653 }
3654 return rc;
3655}
3656
3657
3658/**
3659 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3660 *
3661 * @returns VBox status code.
3662 * @param pVCpu Pointer to the VMCPU.
3663 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3664 * out-of-sync. Make sure to update the required fields
3665 * before using them.
3666 *
3667 * @remarks No-long-jump zone!!!
3668 */
3669DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3670{
3671 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3672 AssertRCReturn(rc, rc);
3673 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3674 AssertRCReturn(rc, rc);
3675 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 return rc;
3678}
3679
3680
3681/**
3682 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3683 * CR0 is partially shared with the host and we have to consider the FPU bits.
3684 *
3685 * @returns VBox status code.
3686 * @param pVM Pointer to the VM.
3687 * @param pVCpu Pointer to the VMCPU.
3688 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3689 * out-of-sync. Make sure to update the required fields
3690 * before using them.
3691 *
3692 * @remarks No-long-jump zone!!!
3693 */
3694static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3695{
3696 /*
3697 * Guest CR0.
3698 * Guest FPU.
3699 */
3700 int rc = VINF_SUCCESS;
3701 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3702 {
3703 Assert(!(pMixedCtx->cr0 >> 32));
3704 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3705 PVM pVM = pVCpu->CTX_SUFF(pVM);
3706
3707 /* The guest's view (read access) of its CR0 is unblemished. */
3708 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3709 AssertRCReturn(rc, rc);
3710 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3711
3712 /* Setup VT-x's view of the guest CR0. */
3713 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3714 if (pVM->hm.s.fNestedPaging)
3715 {
3716 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3717 {
3718 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3719 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3720 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3721 }
3722 else
3723 {
3724 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3725 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3726 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3727 }
3728
3729 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3730 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3731 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3732
3733 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3734 AssertRCReturn(rc, rc);
3735 }
3736 else
3737 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3738
3739 /*
3740 * Guest FPU bits.
3741 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3742 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3743 */
3744 u32GuestCR0 |= X86_CR0_NE;
3745 bool fInterceptNM = false;
3746 if (CPUMIsGuestFPUStateActive(pVCpu))
3747 {
3748 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3749 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3750 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3751 }
3752 else
3753 {
3754 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3755 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3756 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3757 }
3758
3759 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3760 bool fInterceptMF = false;
3761 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3762 fInterceptMF = true;
3763
3764 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3765 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3766 {
3767 Assert(PDMVmmDevHeapIsEnabled(pVM));
3768 Assert(pVM->hm.s.vmx.pRealModeTSS);
3769 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3770 fInterceptNM = true;
3771 fInterceptMF = true;
3772 }
3773 else
3774 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3775
3776 if (fInterceptNM)
3777 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3778 else
3779 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3780
3781 if (fInterceptMF)
3782 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3783 else
3784 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3785
3786 /* Additional intercepts for debugging, define these yourself explicitly. */
3787#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3789 | RT_BIT(X86_XCPT_BP)
3790 | RT_BIT(X86_XCPT_DB)
3791 | RT_BIT(X86_XCPT_DE)
3792 | RT_BIT(X86_XCPT_NM)
3793 | RT_BIT(X86_XCPT_TS)
3794 | RT_BIT(X86_XCPT_UD)
3795 | RT_BIT(X86_XCPT_NP)
3796 | RT_BIT(X86_XCPT_SS)
3797 | RT_BIT(X86_XCPT_GP)
3798 | RT_BIT(X86_XCPT_PF)
3799 | RT_BIT(X86_XCPT_MF)
3800 ;
3801#elif defined(HMVMX_ALWAYS_TRAP_PF)
3802 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3803#endif
3804
3805 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3806
3807 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3808 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3809 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3810 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3811 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3812 else
3813 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3814
3815 u32GuestCR0 |= uSetCR0;
3816 u32GuestCR0 &= uZapCR0;
3817 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3818
3819 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3820 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3821 AssertRCReturn(rc, rc);
3822 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3823 AssertRCReturn(rc, rc);
3824 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3825 uZapCR0));
3826
3827 /*
3828 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3829 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3830 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3831 */
3832 uint32_t u32CR0Mask = 0;
3833 u32CR0Mask = X86_CR0_PE
3834 | X86_CR0_NE
3835 | X86_CR0_WP
3836 | X86_CR0_PG
3837 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3838 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3839 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3840
3841 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3842 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3843 * and @bugref{6944}. */
3844#if 0
3845 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3846 u32CR0Mask &= ~X86_CR0_PE;
3847#endif
3848 if (pVM->hm.s.fNestedPaging)
3849 u32CR0Mask &= ~X86_CR0_WP;
3850
3851 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3852 if (fInterceptNM)
3853 {
3854 u32CR0Mask |= X86_CR0_TS
3855 | X86_CR0_MP;
3856 }
3857
3858 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3859 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3860 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3861 AssertRCReturn(rc, rc);
3862 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3863
3864 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3865 }
3866 return rc;
3867}
3868
3869
3870/**
3871 * Loads the guest control registers (CR3, CR4) into the guest-state area
3872 * in the VMCS.
3873 *
3874 * @returns VBox status code.
3875 * @param pVM Pointer to the VM.
3876 * @param pVCpu Pointer to the VMCPU.
3877 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3878 * out-of-sync. Make sure to update the required fields
3879 * before using them.
3880 *
3881 * @remarks No-long-jump zone!!!
3882 */
3883static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3884{
3885 int rc = VINF_SUCCESS;
3886 PVM pVM = pVCpu->CTX_SUFF(pVM);
3887
3888 /*
3889 * Guest CR2.
3890 * It's always loaded in the assembler code. Nothing to do here.
3891 */
3892
3893 /*
3894 * Guest CR3.
3895 */
3896 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3897 {
3898 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3899 if (pVM->hm.s.fNestedPaging)
3900 {
3901 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3902
3903 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3904 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3905 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3906 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3907
3908 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3909 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3910 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3911
3912 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3913 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3914 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3915 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3916
3917 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3918 AssertRCReturn(rc, rc);
3919 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3920
3921 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3922 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3923 {
3924 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3925 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3926 {
3927 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3928 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3929 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3930 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3932 }
3933
3934 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3935 have Unrestricted Execution to handle the guest when it's not using paging. */
3936 GCPhysGuestCR3 = pMixedCtx->cr3;
3937 }
3938 else
3939 {
3940 /*
3941 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3942 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3943 * EPT takes care of translating it to host-physical addresses.
3944 */
3945 RTGCPHYS GCPhys;
3946 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3947 Assert(PDMVmmDevHeapIsEnabled(pVM));
3948
3949 /* We obtain it here every time as the guest could have relocated this PCI region. */
3950 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3951 AssertRCReturn(rc, rc);
3952
3953 GCPhysGuestCR3 = GCPhys;
3954 }
3955
3956 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3957 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3958 }
3959 else
3960 {
3961 /* Non-nested paging case, just use the hypervisor's CR3. */
3962 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3963
3964 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3965 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3966 }
3967 AssertRCReturn(rc, rc);
3968
3969 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3970 }
3971
3972 /*
3973 * Guest CR4.
3974 */
3975 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3976 {
3977 Assert(!(pMixedCtx->cr4 >> 32));
3978 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3979
3980 /* The guest's view of its CR4 is unblemished. */
3981 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3982 AssertRCReturn(rc, rc);
3983 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3984
3985 /* Setup VT-x's view of the guest CR4. */
3986 /*
3987 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3988 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3989 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3990 */
3991 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3992 {
3993 Assert(pVM->hm.s.vmx.pRealModeTSS);
3994 Assert(PDMVmmDevHeapIsEnabled(pVM));
3995 u32GuestCR4 &= ~X86_CR4_VME;
3996 }
3997
3998 if (pVM->hm.s.fNestedPaging)
3999 {
4000 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4001 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4002 {
4003 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4004 u32GuestCR4 |= X86_CR4_PSE;
4005 /* Our identity mapping is a 32-bit page directory. */
4006 u32GuestCR4 &= ~X86_CR4_PAE;
4007 }
4008 /* else use guest CR4.*/
4009 }
4010 else
4011 {
4012 /*
4013 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4014 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4015 */
4016 switch (pVCpu->hm.s.enmShadowMode)
4017 {
4018 case PGMMODE_REAL: /* Real-mode. */
4019 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4020 case PGMMODE_32_BIT: /* 32-bit paging. */
4021 {
4022 u32GuestCR4 &= ~X86_CR4_PAE;
4023 break;
4024 }
4025
4026 case PGMMODE_PAE: /* PAE paging. */
4027 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4028 {
4029 u32GuestCR4 |= X86_CR4_PAE;
4030 break;
4031 }
4032
4033 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4034 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4035#ifdef VBOX_ENABLE_64_BITS_GUESTS
4036 break;
4037#endif
4038 default:
4039 AssertFailed();
4040 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4041 }
4042 }
4043
4044 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4045 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4046 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4047 u32GuestCR4 |= uSetCR4;
4048 u32GuestCR4 &= uZapCR4;
4049
4050 /* Write VT-x's view of the guest CR4 into the VMCS. */
4051 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4052 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4053 AssertRCReturn(rc, rc);
4054
4055 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4056 uint32_t u32CR4Mask = 0;
4057 u32CR4Mask = X86_CR4_VME
4058 | X86_CR4_PAE
4059 | X86_CR4_PGE
4060 | X86_CR4_PSE
4061 | X86_CR4_VMXE;
4062 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4063 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4064 AssertRCReturn(rc, rc);
4065
4066 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4067 }
4068 return rc;
4069}
4070
4071
4072/**
4073 * Loads the guest debug registers into the guest-state area in the VMCS.
4074 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4075 *
4076 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4077 *
4078 * @returns VBox status code.
4079 * @param pVCpu Pointer to the VMCPU.
4080 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4081 * out-of-sync. Make sure to update the required fields
4082 * before using them.
4083 *
4084 * @remarks No-long-jump zone!!!
4085 */
4086static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4087{
4088 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4089 return VINF_SUCCESS;
4090
4091#ifdef VBOX_STRICT
4092 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4093 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4094 {
4095 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4096 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4097 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4098 }
4099#endif
4100
4101 int rc;
4102 PVM pVM = pVCpu->CTX_SUFF(pVM);
4103 bool fInterceptDB = false;
4104 bool fInterceptMovDRx = false;
4105 if ( pVCpu->hm.s.fSingleInstruction
4106 || DBGFIsStepping(pVCpu))
4107 {
4108 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4109 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4110 {
4111 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4112 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4113 AssertRCReturn(rc, rc);
4114 Assert(fInterceptDB == false);
4115 }
4116 else
4117 {
4118 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4119 pVCpu->hm.s.fClearTrapFlag = true;
4120 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4121 fInterceptDB = true;
4122 }
4123 }
4124
4125 if ( fInterceptDB
4126 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4127 {
4128 /*
4129 * Use the combined guest and host DRx values found in the hypervisor
4130 * register set because the debugger has breakpoints active or someone
4131 * is single stepping on the host side without a monitor trap flag.
4132 *
4133 * Note! DBGF expects a clean DR6 state before executing guest code.
4134 */
4135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4136 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4137 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4138 {
4139 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4140 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4141 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4142 }
4143 else
4144#endif
4145 if (!CPUMIsHyperDebugStateActive(pVCpu))
4146 {
4147 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4148 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4149 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4150 }
4151
4152 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4153 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4154 AssertRCReturn(rc, rc);
4155
4156 pVCpu->hm.s.fUsingHyperDR7 = true;
4157 fInterceptDB = true;
4158 fInterceptMovDRx = true;
4159 }
4160 else
4161 {
4162 /*
4163 * If the guest has enabled debug registers, we need to load them prior to
4164 * executing guest code so they'll trigger at the right time.
4165 */
4166 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4167 {
4168#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4169 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4170 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4171 {
4172 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4173 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4174 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4175 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4176 }
4177 else
4178#endif
4179 if (!CPUMIsGuestDebugStateActive(pVCpu))
4180 {
4181 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4182 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4183 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4184 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4185 }
4186 Assert(!fInterceptDB);
4187 Assert(!fInterceptMovDRx);
4188 }
4189 /*
4190 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4191 * must intercept #DB in order to maintain a correct DR6 guest value.
4192 */
4193#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4194 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4195 && !CPUMIsGuestDebugStateActive(pVCpu))
4196#else
4197 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4198#endif
4199 {
4200 fInterceptMovDRx = true;
4201 fInterceptDB = true;
4202 }
4203
4204 /* Update guest DR7. */
4205 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4206 AssertRCReturn(rc, rc);
4207
4208 pVCpu->hm.s.fUsingHyperDR7 = false;
4209 }
4210
4211 /*
4212 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4213 */
4214 if (fInterceptDB)
4215 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4216 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4217 {
4218#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4219 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4220#endif
4221 }
4222 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4223 AssertRCReturn(rc, rc);
4224
4225 /*
4226 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4227 */
4228 if (fInterceptMovDRx)
4229 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4230 else
4231 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4233 AssertRCReturn(rc, rc);
4234
4235 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4236 return VINF_SUCCESS;
4237}
4238
4239
4240#ifdef VBOX_STRICT
4241/**
4242 * Strict function to validate segment registers.
4243 *
4244 * @remarks ASSUMES CR0 is up to date.
4245 */
4246static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4247{
4248 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4249 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4250 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4251 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4252 && ( !CPUMIsGuestInRealModeEx(pCtx)
4253 && !CPUMIsGuestInV86ModeEx(pCtx)))
4254 {
4255 /* Protected mode checks */
4256 /* CS */
4257 Assert(pCtx->cs.Attr.n.u1Present);
4258 Assert(!(pCtx->cs.Attr.u & 0xf00));
4259 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->cs.Attr.n.u1Granularity));
4262 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4263 || (pCtx->cs.Attr.n.u1Granularity));
4264 /* CS cannot be loaded with NULL in protected mode. */
4265 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4266 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4267 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4268 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4269 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4270 else
4271 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4272 /* SS */
4273 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4274 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4275 if ( !(pCtx->cr0 & X86_CR0_PE)
4276 || pCtx->cs.Attr.n.u4Type == 3)
4277 {
4278 Assert(!pCtx->ss.Attr.n.u2Dpl);
4279 }
4280 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4281 {
4282 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4283 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4284 Assert(pCtx->ss.Attr.n.u1Present);
4285 Assert(!(pCtx->ss.Attr.u & 0xf00));
4286 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4287 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4288 || !(pCtx->ss.Attr.n.u1Granularity));
4289 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4290 || (pCtx->ss.Attr.n.u1Granularity));
4291 }
4292 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4293 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4294 {
4295 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4296 Assert(pCtx->ds.Attr.n.u1Present);
4297 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4298 Assert(!(pCtx->ds.Attr.u & 0xf00));
4299 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4300 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4301 || !(pCtx->ds.Attr.n.u1Granularity));
4302 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4303 || (pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4305 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4306 }
4307 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4308 {
4309 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4310 Assert(pCtx->es.Attr.n.u1Present);
4311 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4312 Assert(!(pCtx->es.Attr.u & 0xf00));
4313 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4314 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4315 || !(pCtx->es.Attr.n.u1Granularity));
4316 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4317 || (pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4319 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4320 }
4321 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4322 {
4323 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4324 Assert(pCtx->fs.Attr.n.u1Present);
4325 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4326 Assert(!(pCtx->fs.Attr.u & 0xf00));
4327 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4328 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4329 || !(pCtx->fs.Attr.n.u1Granularity));
4330 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4331 || (pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4333 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4334 }
4335 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4336 {
4337 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4338 Assert(pCtx->gs.Attr.n.u1Present);
4339 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4340 Assert(!(pCtx->gs.Attr.u & 0xf00));
4341 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4342 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4343 || !(pCtx->gs.Attr.n.u1Granularity));
4344 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4345 || (pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4347 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4348 }
4349 /* 64-bit capable CPUs. */
4350# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4351 if (HMVMX_IS_64BIT_HOST_MODE())
4352 {
4353 Assert(!(pCtx->cs.u64Base >> 32));
4354 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4355 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4356 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4357 }
4358# endif
4359 }
4360 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4361 || ( CPUMIsGuestInRealModeEx(pCtx)
4362 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4363 {
4364 /* Real and v86 mode checks. */
4365 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4366 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4367 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4368 {
4369 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4370 }
4371 else
4372 {
4373 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4374 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4375 }
4376
4377 /* CS */
4378 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4379 Assert(pCtx->cs.u32Limit == 0xffff);
4380 Assert(u32CSAttr == 0xf3);
4381 /* SS */
4382 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4383 Assert(pCtx->ss.u32Limit == 0xffff);
4384 Assert(u32SSAttr == 0xf3);
4385 /* DS */
4386 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4387 Assert(pCtx->ds.u32Limit == 0xffff);
4388 Assert(u32DSAttr == 0xf3);
4389 /* ES */
4390 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4391 Assert(pCtx->es.u32Limit == 0xffff);
4392 Assert(u32ESAttr == 0xf3);
4393 /* FS */
4394 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4395 Assert(pCtx->fs.u32Limit == 0xffff);
4396 Assert(u32FSAttr == 0xf3);
4397 /* GS */
4398 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4399 Assert(pCtx->gs.u32Limit == 0xffff);
4400 Assert(u32GSAttr == 0xf3);
4401 /* 64-bit capable CPUs. */
4402# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4403 if (HMVMX_IS_64BIT_HOST_MODE())
4404 {
4405 Assert(!(pCtx->cs.u64Base >> 32));
4406 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4407 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4408 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4409 }
4410# endif
4411 }
4412}
4413#endif /* VBOX_STRICT */
4414
4415
4416/**
4417 * Writes a guest segment register into the guest-state area in the VMCS.
4418 *
4419 * @returns VBox status code.
4420 * @param pVCpu Pointer to the VMCPU.
4421 * @param idxSel Index of the selector in the VMCS.
4422 * @param idxLimit Index of the segment limit in the VMCS.
4423 * @param idxBase Index of the segment base in the VMCS.
4424 * @param idxAccess Index of the access rights of the segment in the VMCS.
4425 * @param pSelReg Pointer to the segment selector.
4426 *
4427 * @remarks No-long-jump zone!!!
4428 */
4429static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4430 uint32_t idxAccess, PCPUMSELREG pSelReg)
4431{
4432 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4433 AssertRCReturn(rc, rc);
4434 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4435 AssertRCReturn(rc, rc);
4436 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVM Pointer to the VM.
4475 * @param pVCPU Pointer to the VMCPU.
4476 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4477 * out-of-sync. Make sure to update the required fields
4478 * before using them.
4479 *
4480 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4481 * @remarks No-long-jump zone!!!
4482 */
4483static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4484{
4485 int rc = VERR_INTERNAL_ERROR_5;
4486 PVM pVM = pVCpu->CTX_SUFF(pVM);
4487
4488 /*
4489 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4490 */
4491 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4492 {
4493 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4494 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4495 {
4496 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4502 }
4503
4504#ifdef VBOX_WITH_REM
4505 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4506 {
4507 Assert(pVM->hm.s.vmx.pRealModeTSS);
4508 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4509 if ( pVCpu->hm.s.vmx.fWasInRealMode
4510 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4511 {
4512 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4513 in real-mode (e.g. OpenBSD 4.0) */
4514 REMFlushTBs(pVM);
4515 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4516 pVCpu->hm.s.vmx.fWasInRealMode = false;
4517 }
4518 }
4519#endif
4520 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4521 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4522 AssertRCReturn(rc, rc);
4523 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4524 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4525 AssertRCReturn(rc, rc);
4526 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4527 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4528 AssertRCReturn(rc, rc);
4529 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4530 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4531 AssertRCReturn(rc, rc);
4532 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4533 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4534 AssertRCReturn(rc, rc);
4535 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4536 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4537 AssertRCReturn(rc, rc);
4538
4539#ifdef VBOX_STRICT
4540 /* Validate. */
4541 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4542#endif
4543
4544 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4545 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4546 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4547 }
4548
4549 /*
4550 * Guest TR.
4551 */
4552 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4553 {
4554 /*
4555 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4556 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4557 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4558 */
4559 uint16_t u16Sel = 0;
4560 uint32_t u32Limit = 0;
4561 uint64_t u64Base = 0;
4562 uint32_t u32AccessRights = 0;
4563
4564 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565 {
4566 u16Sel = pMixedCtx->tr.Sel;
4567 u32Limit = pMixedCtx->tr.u32Limit;
4568 u64Base = pMixedCtx->tr.u64Base;
4569 u32AccessRights = pMixedCtx->tr.Attr.u;
4570 }
4571 else
4572 {
4573 Assert(pVM->hm.s.vmx.pRealModeTSS);
4574 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4575
4576 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4577 RTGCPHYS GCPhys;
4578 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4579 AssertRCReturn(rc, rc);
4580
4581 X86DESCATTR DescAttr;
4582 DescAttr.u = 0;
4583 DescAttr.n.u1Present = 1;
4584 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4585
4586 u16Sel = 0;
4587 u32Limit = HM_VTX_TSS_SIZE;
4588 u64Base = GCPhys; /* in real-mode phys = virt. */
4589 u32AccessRights = DescAttr.u;
4590 }
4591
4592 /* Validate. */
4593 Assert(!(u16Sel & RT_BIT(2)));
4594 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4595 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4596 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4597 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4598 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4599 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4600 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4601 Assert( (u32Limit & 0xfff) == 0xfff
4602 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4603 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4604 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4605
4606 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4607 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4608 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4621 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4622
4623 /* Validate. */
4624 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4625
4626 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4627 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4628 }
4629
4630 /*
4631 * Guest LDTR.
4632 */
4633 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4634 {
4635 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4636 uint32_t u32Access = 0;
4637 if (!pMixedCtx->ldtr.Attr.u)
4638 u32Access = X86DESCATTR_UNUSABLE;
4639 else
4640 u32Access = pMixedCtx->ldtr.Attr.u;
4641
4642 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4643 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4644 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4645 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4646
4647 /* Validate. */
4648 if (!(u32Access & X86DESCATTR_UNUSABLE))
4649 {
4650 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4651 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4652 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4655 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4656 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4657 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4658 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4659 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4660 }
4661
4662 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4663 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4664 }
4665
4666 /*
4667 * Guest IDTR.
4668 */
4669 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4670 {
4671 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4672 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4673
4674 /* Validate. */
4675 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4676
4677 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4678 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4679 }
4680
4681 return VINF_SUCCESS;
4682}
4683
4684
4685/**
4686 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4687 * areas. These MSRs will automatically be loaded to the host CPU on every
4688 * successful VM entry and stored from the host CPU on every successful VM-exit.
4689 *
4690 * This also creates/updates MSR slots for the host MSRs. The actual host
4691 * MSR values are -not- updated here for performance reasons. See
4692 * hmR0VmxSaveHostMsrs().
4693 *
4694 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4695 *
4696 * @returns VBox status code.
4697 * @param pVCpu Pointer to the VMCPU.
4698 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4699 * out-of-sync. Make sure to update the required fields
4700 * before using them.
4701 *
4702 * @remarks No-long-jump zone!!!
4703 */
4704static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4705{
4706 AssertPtr(pVCpu);
4707 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4708
4709 /*
4710 * MSRs that we use the auto-load/store MSR area in the VMCS.
4711 */
4712 PVM pVM = pVCpu->CTX_SUFF(pVM);
4713 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4714 {
4715 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4716#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4717 if (pVM->hm.s.fAllow64BitGuests)
4718 {
4719 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4720 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4721 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4722 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4723# ifdef DEBUG
4724 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4725 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4726 {
4727 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4728 pMsr->u64Value));
4729 }
4730# endif
4731 }
4732#endif
4733 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4734 }
4735
4736 /*
4737 * Guest Sysenter MSRs.
4738 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4739 * VM-exits on WRMSRs for these MSRs.
4740 */
4741 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4742 {
4743 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4744 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4745 }
4746
4747 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4748 {
4749 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4750 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4751 }
4752
4753 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4754 {
4755 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4756 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4757 }
4758
4759 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4760 {
4761 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4762 {
4763 /*
4764 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4765 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4766 */
4767 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4768 {
4769 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4770 AssertRCReturn(rc,rc);
4771 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4772 }
4773 else
4774 {
4775 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4776 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4777 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4778 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4779 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4780 }
4781 }
4782 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4783 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4784 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4785 }
4786
4787 return VINF_SUCCESS;
4788}
4789
4790
4791/**
4792 * Loads the guest activity state into the guest-state area in the VMCS.
4793 *
4794 * @returns VBox status code.
4795 * @param pVCpu Pointer to the VMCPU.
4796 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4797 * out-of-sync. Make sure to update the required fields
4798 * before using them.
4799 *
4800 * @remarks No-long-jump zone!!!
4801 */
4802static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4803{
4804 NOREF(pCtx);
4805 /** @todo See if we can make use of other states, e.g.
4806 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4807 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4808 {
4809 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4810 AssertRCReturn(rc, rc);
4811
4812 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4813 }
4814 return VINF_SUCCESS;
4815}
4816
4817
4818/**
4819 * Sets up the appropriate function to run guest code.
4820 *
4821 * @returns VBox status code.
4822 * @param pVCpu Pointer to the VMCPU.
4823 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4824 * out-of-sync. Make sure to update the required fields
4825 * before using them.
4826 *
4827 * @remarks No-long-jump zone!!!
4828 */
4829static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4830{
4831 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4832 {
4833#ifndef VBOX_ENABLE_64_BITS_GUESTS
4834 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4835#endif
4836 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4837#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4838 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4839 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4840 {
4841 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4842 {
4843 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4844 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4845 | HM_CHANGED_VMX_EXIT_CTLS
4846 | HM_CHANGED_VMX_ENTRY_CTLS
4847 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4848 }
4849 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4850 }
4851#else
4852 /* 64-bit host or hybrid host. */
4853 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4854#endif
4855 }
4856 else
4857 {
4858 /* Guest is not in long mode, use the 32-bit handler. */
4859#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4860 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4861 {
4862 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4863 {
4864 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4865 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4866 | HM_CHANGED_VMX_EXIT_CTLS
4867 | HM_CHANGED_VMX_ENTRY_CTLS
4868 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4869 }
4870 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4871 }
4872#else
4873 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4874#endif
4875 }
4876 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4877 return VINF_SUCCESS;
4878}
4879
4880
4881/**
4882 * Wrapper for running the guest code in VT-x.
4883 *
4884 * @returns VBox strict status code.
4885 * @param pVM Pointer to the VM.
4886 * @param pVCpu Pointer to the VMCPU.
4887 * @param pCtx Pointer to the guest-CPU context.
4888 *
4889 * @remarks No-long-jump zone!!!
4890 */
4891DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4892{
4893 /*
4894 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4895 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4896 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4897 */
4898 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4899 /** @todo Add stats for resume vs launch. */
4900#ifdef VBOX_WITH_KERNEL_USING_XMM
4901 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4902#else
4903 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4904#endif
4905}
4906
4907
4908/**
4909 * Reports world-switch error and dumps some useful debug info.
4910 *
4911 * @param pVM Pointer to the VM.
4912 * @param pVCpu Pointer to the VMCPU.
4913 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4914 * @param pCtx Pointer to the guest-CPU context.
4915 * @param pVmxTransient Pointer to the VMX transient structure (only
4916 * exitReason updated).
4917 */
4918static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4919{
4920 Assert(pVM);
4921 Assert(pVCpu);
4922 Assert(pCtx);
4923 Assert(pVmxTransient);
4924 HMVMX_ASSERT_PREEMPT_SAFE();
4925
4926 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4927 switch (rcVMRun)
4928 {
4929 case VERR_VMX_INVALID_VMXON_PTR:
4930 AssertFailed();
4931 break;
4932 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4933 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4934 {
4935 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4936 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4937 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4938 AssertRC(rc);
4939
4940 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4941 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4942 Cannot do it here as we may have been long preempted. */
4943
4944#ifdef VBOX_STRICT
4945 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4946 pVmxTransient->uExitReason));
4947 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4948 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4949 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4950 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4951 else
4952 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4953 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4954 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4955
4956 /* VMX control bits. */
4957 uint32_t u32Val;
4958 uint64_t u64Val;
4959 HMVMXHCUINTREG uHCReg;
4960 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4961 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4992 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4993 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5000 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5001 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5002
5003 /* Guest bits. */
5004 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5005 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5007 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5008 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5009 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5010 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5011 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5012
5013 /* Host bits. */
5014 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5015 Log4(("Host CR0 %#RHr\n", uHCReg));
5016 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5017 Log4(("Host CR3 %#RHr\n", uHCReg));
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR4 %#RHr\n", uHCReg));
5020
5021 RTGDTR HostGdtr;
5022 PCX86DESCHC pDesc;
5023 ASMGetGDTR(&HostGdtr);
5024 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5025 Log4(("Host CS %#08x\n", u32Val));
5026 if (u32Val < HostGdtr.cbGdt)
5027 {
5028 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5029 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5030 }
5031
5032 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5033 Log4(("Host DS %#08x\n", u32Val));
5034 if (u32Val < HostGdtr.cbGdt)
5035 {
5036 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5037 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5038 }
5039
5040 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5041 Log4(("Host ES %#08x\n", u32Val));
5042 if (u32Val < HostGdtr.cbGdt)
5043 {
5044 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5045 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5046 }
5047
5048 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5049 Log4(("Host FS %#08x\n", u32Val));
5050 if (u32Val < HostGdtr.cbGdt)
5051 {
5052 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5053 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5054 }
5055
5056 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5057 Log4(("Host GS %#08x\n", u32Val));
5058 if (u32Val < HostGdtr.cbGdt)
5059 {
5060 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5061 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5062 }
5063
5064 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5065 Log4(("Host SS %#08x\n", u32Val));
5066 if (u32Val < HostGdtr.cbGdt)
5067 {
5068 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5069 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5070 }
5071
5072 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5073 Log4(("Host TR %#08x\n", u32Val));
5074 if (u32Val < HostGdtr.cbGdt)
5075 {
5076 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5077 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5078 }
5079
5080 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5081 Log4(("Host TR Base %#RHv\n", uHCReg));
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5083 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5087 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5089 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5091 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5093 Log4(("Host RSP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5095 Log4(("Host RIP %#RHv\n", uHCReg));
5096# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5097 if (HMVMX_IS_64BIT_HOST_MODE())
5098 {
5099 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5100 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5101 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5102 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5103 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5104 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5105 }
5106# endif
5107#endif /* VBOX_STRICT */
5108 break;
5109 }
5110
5111 default:
5112 /* Impossible */
5113 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5114 break;
5115 }
5116 NOREF(pVM); NOREF(pCtx);
5117}
5118
5119
5120#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5121#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5122# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5123#endif
5124#ifdef VBOX_STRICT
5125static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5126{
5127 switch (idxField)
5128 {
5129 case VMX_VMCS_GUEST_RIP:
5130 case VMX_VMCS_GUEST_RSP:
5131 case VMX_VMCS_GUEST_SYSENTER_EIP:
5132 case VMX_VMCS_GUEST_SYSENTER_ESP:
5133 case VMX_VMCS_GUEST_GDTR_BASE:
5134 case VMX_VMCS_GUEST_IDTR_BASE:
5135 case VMX_VMCS_GUEST_CS_BASE:
5136 case VMX_VMCS_GUEST_DS_BASE:
5137 case VMX_VMCS_GUEST_ES_BASE:
5138 case VMX_VMCS_GUEST_FS_BASE:
5139 case VMX_VMCS_GUEST_GS_BASE:
5140 case VMX_VMCS_GUEST_SS_BASE:
5141 case VMX_VMCS_GUEST_LDTR_BASE:
5142 case VMX_VMCS_GUEST_TR_BASE:
5143 case VMX_VMCS_GUEST_CR3:
5144 return true;
5145 }
5146 return false;
5147}
5148
5149static bool hmR0VmxIsValidReadField(uint32_t idxField)
5150{
5151 switch (idxField)
5152 {
5153 /* Read-only fields. */
5154 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5155 return true;
5156 }
5157 /* Remaining readable fields should also be writable. */
5158 return hmR0VmxIsValidWriteField(idxField);
5159}
5160#endif /* VBOX_STRICT */
5161
5162
5163/**
5164 * Executes the specified handler in 64-bit mode.
5165 *
5166 * @returns VBox status code.
5167 * @param pVM Pointer to the VM.
5168 * @param pVCpu Pointer to the VMCPU.
5169 * @param pCtx Pointer to the guest CPU context.
5170 * @param enmOp The operation to perform.
5171 * @param cbParam Number of parameters.
5172 * @param paParam Array of 32-bit parameters.
5173 */
5174VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5175 uint32_t *paParam)
5176{
5177 int rc, rc2;
5178 PHMGLOBALCPUINFO pCpu;
5179 RTHCPHYS HCPhysCpuPage;
5180 RTCCUINTREG uOldEflags;
5181
5182 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5183 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5184 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5185 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5186
5187#ifdef VBOX_STRICT
5188 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5189 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5190
5191 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5192 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5193#endif
5194
5195 /* Disable interrupts. */
5196 uOldEflags = ASMIntDisableFlags();
5197
5198#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5199 RTCPUID idHostCpu = RTMpCpuId();
5200 CPUMR0SetLApic(pVCpu, idHostCpu);
5201#endif
5202
5203 pCpu = HMR0GetCurrentCpu();
5204 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5205
5206 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5207 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5208
5209 /* Leave VMX Root Mode. */
5210 VMXDisable();
5211
5212 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5213
5214 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5215 CPUMSetHyperEIP(pVCpu, enmOp);
5216 for (int i = (int)cbParam - 1; i >= 0; i--)
5217 CPUMPushHyper(pVCpu, paParam[i]);
5218
5219 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5220
5221 /* Call the switcher. */
5222 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5223 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5224
5225 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5226 /* Make sure the VMX instructions don't cause #UD faults. */
5227 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5228
5229 /* Re-enter VMX Root Mode */
5230 rc2 = VMXEnable(HCPhysCpuPage);
5231 if (RT_FAILURE(rc2))
5232 {
5233 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5234 ASMSetFlags(uOldEflags);
5235 return rc2;
5236 }
5237
5238 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5239 AssertRC(rc2);
5240 Assert(!(ASMGetFlags() & X86_EFL_IF));
5241 ASMSetFlags(uOldEflags);
5242 return rc;
5243}
5244
5245
5246/**
5247 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5248 * supporting 64-bit guests.
5249 *
5250 * @returns VBox status code.
5251 * @param fResume Whether to VMLAUNCH or VMRESUME.
5252 * @param pCtx Pointer to the guest-CPU context.
5253 * @param pCache Pointer to the VMCS cache.
5254 * @param pVM Pointer to the VM.
5255 * @param pVCpu Pointer to the VMCPU.
5256 */
5257DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5258{
5259 uint32_t aParam[6];
5260 PHMGLOBALCPUINFO pCpu = NULL;
5261 RTHCPHYS HCPhysCpuPage = 0;
5262 int rc = VERR_INTERNAL_ERROR_5;
5263
5264 pCpu = HMR0GetCurrentCpu();
5265 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5266
5267#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5268 pCache->uPos = 1;
5269 pCache->interPD = PGMGetInterPaeCR3(pVM);
5270 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5271#endif
5272
5273#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5274 pCache->TestIn.HCPhysCpuPage = 0;
5275 pCache->TestIn.HCPhysVmcs = 0;
5276 pCache->TestIn.pCache = 0;
5277 pCache->TestOut.HCPhysVmcs = 0;
5278 pCache->TestOut.pCache = 0;
5279 pCache->TestOut.pCtx = 0;
5280 pCache->TestOut.eflags = 0;
5281#endif
5282
5283 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5284 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5285 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5286 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5287 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5288 aParam[5] = 0;
5289
5290#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5291 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5292 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5293#endif
5294 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5295
5296#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5297 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5298 Assert(pCtx->dr[4] == 10);
5299 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5300#endif
5301
5302#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5303 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5304 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5305 pVCpu->hm.s.vmx.HCPhysVmcs));
5306 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5307 pCache->TestOut.HCPhysVmcs));
5308 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5309 pCache->TestOut.pCache));
5310 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5311 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5312 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5313 pCache->TestOut.pCtx));
5314 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5315#endif
5316 return rc;
5317}
5318
5319
5320/**
5321 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5322 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5323 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5324 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5325 *
5326 * @returns VBox status code.
5327 * @param pVM Pointer to the VM.
5328 * @param pVCpu Pointer to the VMCPU.
5329 */
5330static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5331{
5332#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5333{ \
5334 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5335 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5336 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5337 ++cReadFields; \
5338}
5339
5340 AssertPtr(pVM);
5341 AssertPtr(pVCpu);
5342 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5343 uint32_t cReadFields = 0;
5344
5345 /*
5346 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5347 * and serve to indicate exceptions to the rules.
5348 */
5349
5350 /* Guest-natural selector base fields. */
5351#if 0
5352 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5355#endif
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5368#if 0
5369 /* Unused natural width guest-state fields. */
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5372#endif
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5375
5376 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5377#if 0
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5387#endif
5388
5389 /* Natural width guest-state fields. */
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5391#if 0
5392 /* Currently unused field. */
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5394#endif
5395
5396 if (pVM->hm.s.fNestedPaging)
5397 {
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5399 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5400 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5401 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5402 }
5403 else
5404 {
5405 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5406 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5407 }
5408
5409#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5410 return VINF_SUCCESS;
5411}
5412
5413
5414/**
5415 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5416 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5417 * darwin, running 64-bit guests).
5418 *
5419 * @returns VBox status code.
5420 * @param pVCpu Pointer to the VMCPU.
5421 * @param idxField The VMCS field encoding.
5422 * @param u64Val 16, 32 or 64-bit value.
5423 */
5424VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5425{
5426 int rc;
5427 switch (idxField)
5428 {
5429 /*
5430 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5431 */
5432 /* 64-bit Control fields. */
5433 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5434 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5435 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5436 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5437 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5438 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5439 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5440 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5441 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5442 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5443 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5444 case VMX_VMCS64_CTRL_EPTP_FULL:
5445 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5446 /* 64-bit Guest-state fields. */
5447 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5448 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5449 case VMX_VMCS64_GUEST_PAT_FULL:
5450 case VMX_VMCS64_GUEST_EFER_FULL:
5451 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5452 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5453 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5454 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5455 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5456 /* 64-bit Host-state fields. */
5457 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5458 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5459 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5460 {
5461 rc = VMXWriteVmcs32(idxField, u64Val);
5462 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5463 break;
5464 }
5465
5466 /*
5467 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5468 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5469 */
5470 /* Natural-width Guest-state fields. */
5471 case VMX_VMCS_GUEST_CR3:
5472 case VMX_VMCS_GUEST_ES_BASE:
5473 case VMX_VMCS_GUEST_CS_BASE:
5474 case VMX_VMCS_GUEST_SS_BASE:
5475 case VMX_VMCS_GUEST_DS_BASE:
5476 case VMX_VMCS_GUEST_FS_BASE:
5477 case VMX_VMCS_GUEST_GS_BASE:
5478 case VMX_VMCS_GUEST_LDTR_BASE:
5479 case VMX_VMCS_GUEST_TR_BASE:
5480 case VMX_VMCS_GUEST_GDTR_BASE:
5481 case VMX_VMCS_GUEST_IDTR_BASE:
5482 case VMX_VMCS_GUEST_RSP:
5483 case VMX_VMCS_GUEST_RIP:
5484 case VMX_VMCS_GUEST_SYSENTER_ESP:
5485 case VMX_VMCS_GUEST_SYSENTER_EIP:
5486 {
5487 if (!(u64Val >> 32))
5488 {
5489 /* If this field is 64-bit, VT-x will zero out the top bits. */
5490 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5491 }
5492 else
5493 {
5494 /* Assert that only the 32->64 switcher case should ever come here. */
5495 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5496 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5497 }
5498 break;
5499 }
5500
5501 default:
5502 {
5503 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5504 rc = VERR_INVALID_PARAMETER;
5505 break;
5506 }
5507 }
5508 AssertRCReturn(rc, rc);
5509 return rc;
5510}
5511
5512
5513/**
5514 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5515 * hosts (except darwin) for 64-bit guests.
5516 *
5517 * @param pVCpu Pointer to the VMCPU.
5518 * @param idxField The VMCS field encoding.
5519 * @param u64Val 16, 32 or 64-bit value.
5520 */
5521VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5522{
5523 AssertPtr(pVCpu);
5524 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5525
5526 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5527 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5528
5529 /* Make sure there are no duplicates. */
5530 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5531 {
5532 if (pCache->Write.aField[i] == idxField)
5533 {
5534 pCache->Write.aFieldVal[i] = u64Val;
5535 return VINF_SUCCESS;
5536 }
5537 }
5538
5539 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5540 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5541 pCache->Write.cValidEntries++;
5542 return VINF_SUCCESS;
5543}
5544
5545/* Enable later when the assembly code uses these as callbacks. */
5546#if 0
5547/*
5548 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5549 *
5550 * @param pVCpu Pointer to the VMCPU.
5551 * @param pCache Pointer to the VMCS cache.
5552 *
5553 * @remarks No-long-jump zone!!!
5554 */
5555VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5556{
5557 AssertPtr(pCache);
5558 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5559 {
5560 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5561 AssertRC(rc);
5562 }
5563 pCache->Write.cValidEntries = 0;
5564}
5565
5566
5567/**
5568 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5569 *
5570 * @param pVCpu Pointer to the VMCPU.
5571 * @param pCache Pointer to the VMCS cache.
5572 *
5573 * @remarks No-long-jump zone!!!
5574 */
5575VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5576{
5577 AssertPtr(pCache);
5578 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5579 {
5580 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5581 AssertRC(rc);
5582 }
5583}
5584#endif
5585#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5586
5587
5588/**
5589 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5590 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5591 * timer.
5592 *
5593 * @returns VBox status code.
5594 * @param pVCpu Pointer to the VMCPU.
5595 *
5596 * @remarks No-long-jump zone!!!
5597 */
5598static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5599{
5600 int rc = VERR_INTERNAL_ERROR_5;
5601 bool fOffsettedTsc = false;
5602 bool fParavirtTsc = false;
5603 PVM pVM = pVCpu->CTX_SUFF(pVM);
5604 if (pVM->hm.s.vmx.fUsePreemptTimer)
5605 {
5606 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5607 &pVCpu->hm.s.vmx.u64TSCOffset);
5608
5609 /* Make sure the returned values have sane upper and lower boundaries. */
5610 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5611 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5612 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5613 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5614
5615 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5616 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5617 }
5618 else
5619 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5620
5621#if 1
5622 if (fParavirtTsc)
5623 {
5624#if 1
5625 uint64_t const u64CurTsc = ASMReadTSC();
5626 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5627 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5628 {
5629 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5630 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5631 }
5632
5633 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5634#endif
5635 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5636 AssertRC(rc);
5637 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5638 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5639
5640 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5641 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5642 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5643 }
5644 else
5645#else
5646 if (fParavirtTsc)
5647 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5648#endif
5649 if (fOffsettedTsc)
5650 {
5651 uint64_t u64CurTSC = ASMReadTSC();
5652 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5653 {
5654 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5655 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5656
5657 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5658 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5659 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5660 }
5661 else
5662 {
5663 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5664 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5665 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5666 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5667 }
5668 }
5669 else
5670 {
5671 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5672 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5673 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5674 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5675 }
5676}
5677
5678
5679/**
5680 * Determines if an exception is a contributory exception. Contributory
5681 * exceptions are ones which can cause double-faults. Page-fault is
5682 * intentionally not included here as it's a conditional contributory exception.
5683 *
5684 * @returns true if the exception is contributory, false otherwise.
5685 * @param uVector The exception vector.
5686 */
5687DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5688{
5689 switch (uVector)
5690 {
5691 case X86_XCPT_GP:
5692 case X86_XCPT_SS:
5693 case X86_XCPT_NP:
5694 case X86_XCPT_TS:
5695 case X86_XCPT_DE:
5696 return true;
5697 default:
5698 break;
5699 }
5700 return false;
5701}
5702
5703
5704/**
5705 * Sets an event as a pending event to be injected into the guest.
5706 *
5707 * @param pVCpu Pointer to the VMCPU.
5708 * @param u32IntInfo The VM-entry interruption-information field.
5709 * @param cbInstr The VM-entry instruction length in bytes (for software
5710 * interrupts, exceptions and privileged software
5711 * exceptions).
5712 * @param u32ErrCode The VM-entry exception error code.
5713 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5714 * page-fault.
5715 *
5716 * @remarks Statistics counter assumes this is a guest event being injected or
5717 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5718 * always incremented.
5719 */
5720DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5721 RTGCUINTPTR GCPtrFaultAddress)
5722{
5723 Assert(!pVCpu->hm.s.Event.fPending);
5724 pVCpu->hm.s.Event.fPending = true;
5725 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5726 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5727 pVCpu->hm.s.Event.cbInstr = cbInstr;
5728 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5729
5730 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5731}
5732
5733
5734/**
5735 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5736 *
5737 * @param pVCpu Pointer to the VMCPU.
5738 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5739 * out-of-sync. Make sure to update the required fields
5740 * before using them.
5741 */
5742DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5743{
5744 NOREF(pMixedCtx);
5745 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5746 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5747 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5748 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5749}
5750
5751
5752/**
5753 * Handle a condition that occurred while delivering an event through the guest
5754 * IDT.
5755 *
5756 * @returns VBox status code (informational error codes included).
5757 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5758 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5759 * continue execution of the guest which will delivery the #DF.
5760 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5761 *
5762 * @param pVCpu Pointer to the VMCPU.
5763 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5764 * out-of-sync. Make sure to update the required fields
5765 * before using them.
5766 * @param pVmxTransient Pointer to the VMX transient structure.
5767 *
5768 * @remarks No-long-jump zone!!!
5769 */
5770static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5771{
5772 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5773
5774 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5775 AssertRCReturn(rc, rc);
5776 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5777 {
5778 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5779 AssertRCReturn(rc, rc);
5780
5781 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5782 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5783
5784 typedef enum
5785 {
5786 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5787 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5788 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5789 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5790 } VMXREFLECTXCPT;
5791
5792 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5793 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5794 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5795 {
5796 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5797 {
5798 enmReflect = VMXREFLECTXCPT_XCPT;
5799#ifdef VBOX_STRICT
5800 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5801 && uExitVector == X86_XCPT_PF)
5802 {
5803 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5804 }
5805#endif
5806 if ( uExitVector == X86_XCPT_PF
5807 && uIdtVector == X86_XCPT_PF)
5808 {
5809 pVmxTransient->fVectoringPF = true;
5810 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5811 }
5812 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5813 && hmR0VmxIsContributoryXcpt(uExitVector)
5814 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5815 || uIdtVector == X86_XCPT_PF))
5816 {
5817 enmReflect = VMXREFLECTXCPT_DF;
5818 }
5819 else if (uIdtVector == X86_XCPT_DF)
5820 enmReflect = VMXREFLECTXCPT_TF;
5821 }
5822 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5823 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5824 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5825 {
5826 /*
5827 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5828 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5829 */
5830 enmReflect = VMXREFLECTXCPT_XCPT;
5831 }
5832 }
5833 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5834 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5835 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5836 {
5837 /*
5838 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5839 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5840 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5841 */
5842 enmReflect = VMXREFLECTXCPT_XCPT;
5843 }
5844
5845 /*
5846 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5847 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5848 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5849 *
5850 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5851 */
5852 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5853 && enmReflect == VMXREFLECTXCPT_XCPT
5854 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5855 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5856 {
5857 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5858 }
5859
5860 switch (enmReflect)
5861 {
5862 case VMXREFLECTXCPT_XCPT:
5863 {
5864 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5865 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5866 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5867
5868 uint32_t u32ErrCode = 0;
5869 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5870 {
5871 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5872 AssertRCReturn(rc, rc);
5873 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5874 }
5875
5876 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5877 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5878 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5879 rc = VINF_SUCCESS;
5880 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5881 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5882
5883 break;
5884 }
5885
5886 case VMXREFLECTXCPT_DF:
5887 {
5888 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5889 rc = VINF_HM_DOUBLE_FAULT;
5890 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5891 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5892
5893 break;
5894 }
5895
5896 case VMXREFLECTXCPT_TF:
5897 {
5898 rc = VINF_EM_RESET;
5899 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5900 uExitVector));
5901 break;
5902 }
5903
5904 default:
5905 Assert(rc == VINF_SUCCESS);
5906 break;
5907 }
5908 }
5909 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5910 && uExitVector != X86_XCPT_DF
5911 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5912 {
5913 /*
5914 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5915 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5916 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5917 */
5918 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5919 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5920 }
5921
5922 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5923 return rc;
5924}
5925
5926
5927/**
5928 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5929 *
5930 * @returns VBox status code.
5931 * @param pVCpu Pointer to the VMCPU.
5932 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5933 * out-of-sync. Make sure to update the required fields
5934 * before using them.
5935 *
5936 * @remarks No-long-jump zone!!!
5937 */
5938static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5939{
5940 NOREF(pMixedCtx);
5941
5942 /*
5943 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5944 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5945 */
5946 VMMRZCallRing3Disable(pVCpu);
5947 HM_DISABLE_PREEMPT_IF_NEEDED();
5948
5949 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5950 {
5951 uint32_t uVal = 0;
5952 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5953 AssertRCReturn(rc, rc);
5954
5955 uint32_t uShadow = 0;
5956 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5957 AssertRCReturn(rc, rc);
5958
5959 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5960 CPUMSetGuestCR0(pVCpu, uVal);
5961 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5962 }
5963
5964 HM_RESTORE_PREEMPT_IF_NEEDED();
5965 VMMRZCallRing3Enable(pVCpu);
5966 return VINF_SUCCESS;
5967}
5968
5969
5970/**
5971 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5972 *
5973 * @returns VBox status code.
5974 * @param pVCpu Pointer to the VMCPU.
5975 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5976 * out-of-sync. Make sure to update the required fields
5977 * before using them.
5978 *
5979 * @remarks No-long-jump zone!!!
5980 */
5981static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5982{
5983 NOREF(pMixedCtx);
5984
5985 int rc = VINF_SUCCESS;
5986 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5987 {
5988 uint32_t uVal = 0;
5989 uint32_t uShadow = 0;
5990 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5991 AssertRCReturn(rc, rc);
5992 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5993 AssertRCReturn(rc, rc);
5994
5995 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5996 CPUMSetGuestCR4(pVCpu, uVal);
5997 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5998 }
5999 return rc;
6000}
6001
6002
6003/**
6004 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6005 *
6006 * @returns VBox status code.
6007 * @param pVCpu Pointer to the VMCPU.
6008 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6009 * out-of-sync. Make sure to update the required fields
6010 * before using them.
6011 *
6012 * @remarks No-long-jump zone!!!
6013 */
6014static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6015{
6016 int rc = VINF_SUCCESS;
6017 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6018 {
6019 uint64_t u64Val = 0;
6020 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6021 AssertRCReturn(rc, rc);
6022
6023 pMixedCtx->rip = u64Val;
6024 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6025 }
6026 return rc;
6027}
6028
6029
6030/**
6031 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6032 *
6033 * @returns VBox status code.
6034 * @param pVCpu Pointer to the VMCPU.
6035 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6036 * out-of-sync. Make sure to update the required fields
6037 * before using them.
6038 *
6039 * @remarks No-long-jump zone!!!
6040 */
6041static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6042{
6043 int rc = VINF_SUCCESS;
6044 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6045 {
6046 uint64_t u64Val = 0;
6047 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6048 AssertRCReturn(rc, rc);
6049
6050 pMixedCtx->rsp = u64Val;
6051 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6052 }
6053 return rc;
6054}
6055
6056
6057/**
6058 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6059 *
6060 * @returns VBox status code.
6061 * @param pVCpu Pointer to the VMCPU.
6062 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6063 * out-of-sync. Make sure to update the required fields
6064 * before using them.
6065 *
6066 * @remarks No-long-jump zone!!!
6067 */
6068static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6069{
6070 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6071 {
6072 uint32_t uVal = 0;
6073 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6074 AssertRCReturn(rc, rc);
6075
6076 pMixedCtx->eflags.u32 = uVal;
6077 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6078 {
6079 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6080 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6081
6082 pMixedCtx->eflags.Bits.u1VM = 0;
6083 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6084 }
6085
6086 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6087 }
6088 return VINF_SUCCESS;
6089}
6090
6091
6092/**
6093 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6094 * guest-CPU context.
6095 */
6096DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6097{
6098 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6099 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6100 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6101 return rc;
6102}
6103
6104
6105/**
6106 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6107 * from the guest-state area in the VMCS.
6108 *
6109 * @param pVCpu Pointer to the VMCPU.
6110 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6111 * out-of-sync. Make sure to update the required fields
6112 * before using them.
6113 *
6114 * @remarks No-long-jump zone!!!
6115 */
6116static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6117{
6118 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6119 {
6120 uint32_t uIntrState = 0;
6121 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6122 AssertRC(rc);
6123
6124 if (!uIntrState)
6125 {
6126 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6127 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6128
6129 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6130 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6131 }
6132 else
6133 {
6134 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6135 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6136 {
6137 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6138 AssertRC(rc);
6139 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6140 AssertRC(rc);
6141
6142 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6143 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6144 }
6145 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6146 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6147
6148 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6149 {
6150 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6151 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6152 }
6153 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6154 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6155 }
6156
6157 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6158 }
6159}
6160
6161
6162/**
6163 * Saves the guest's activity state.
6164 *
6165 * @returns VBox status code.
6166 * @param pVCpu Pointer to the VMCPU.
6167 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6168 * out-of-sync. Make sure to update the required fields
6169 * before using them.
6170 *
6171 * @remarks No-long-jump zone!!!
6172 */
6173static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6174{
6175 NOREF(pMixedCtx);
6176 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6177 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6178 return VINF_SUCCESS;
6179}
6180
6181
6182/**
6183 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6184 * the current VMCS into the guest-CPU context.
6185 *
6186 * @returns VBox status code.
6187 * @param pVCpu Pointer to the VMCPU.
6188 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6189 * out-of-sync. Make sure to update the required fields
6190 * before using them.
6191 *
6192 * @remarks No-long-jump zone!!!
6193 */
6194static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6195{
6196 int rc = VINF_SUCCESS;
6197 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6198 {
6199 uint32_t u32Val = 0;
6200 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6201 pMixedCtx->SysEnter.cs = u32Val;
6202 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6203 }
6204
6205 uint64_t u64Val = 0;
6206 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6207 {
6208 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6209 pMixedCtx->SysEnter.eip = u64Val;
6210 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6211 }
6212 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6213 {
6214 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6215 pMixedCtx->SysEnter.esp = u64Val;
6216 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6217 }
6218 return rc;
6219}
6220
6221
6222/**
6223 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6224 * the CPU back into the guest-CPU context.
6225 *
6226 * @returns VBox status code.
6227 * @param pVCpu Pointer to the VMCPU.
6228 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6229 * out-of-sync. Make sure to update the required fields
6230 * before using them.
6231 *
6232 * @remarks No-long-jump zone!!!
6233 */
6234static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6235{
6236#if HC_ARCH_BITS == 64
6237 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6238 {
6239 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6240 VMMRZCallRing3Disable(pVCpu);
6241 HM_DISABLE_PREEMPT_IF_NEEDED();
6242
6243 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6244 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6245 {
6246 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6247 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6248 }
6249
6250 HM_RESTORE_PREEMPT_IF_NEEDED();
6251 VMMRZCallRing3Enable(pVCpu);
6252 }
6253 else
6254 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6255#else
6256 NOREF(pMixedCtx);
6257 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6258#endif
6259
6260 return VINF_SUCCESS;
6261}
6262
6263
6264/**
6265 * Saves the auto load/store'd guest MSRs from the current VMCS into
6266 * the guest-CPU context.
6267 *
6268 * @returns VBox status code.
6269 * @param pVCpu Pointer to the VMCPU.
6270 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6271 * out-of-sync. Make sure to update the required fields
6272 * before using them.
6273 *
6274 * @remarks No-long-jump zone!!!
6275 */
6276static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6277{
6278 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6279 return VINF_SUCCESS;
6280
6281 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6282 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6283 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6284 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6285 {
6286 switch (pMsr->u32Msr)
6287 {
6288 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6289 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6290 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6291 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6292 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6293 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6294 break;
6295
6296 default:
6297 {
6298 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6299 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6300 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6301 }
6302 }
6303 }
6304
6305 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6306 return VINF_SUCCESS;
6307}
6308
6309
6310/**
6311 * Saves the guest control registers from the current VMCS into the guest-CPU
6312 * context.
6313 *
6314 * @returns VBox status code.
6315 * @param pVCpu Pointer to the VMCPU.
6316 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6317 * out-of-sync. Make sure to update the required fields
6318 * before using them.
6319 *
6320 * @remarks No-long-jump zone!!!
6321 */
6322static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6323{
6324 /* Guest CR0. Guest FPU. */
6325 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6326 AssertRCReturn(rc, rc);
6327
6328 /* Guest CR4. */
6329 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6330 AssertRCReturn(rc, rc);
6331
6332 /* Guest CR2 - updated always during the world-switch or in #PF. */
6333 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6334 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6335 {
6336 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6337 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6338
6339 PVM pVM = pVCpu->CTX_SUFF(pVM);
6340 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6341 || ( pVM->hm.s.fNestedPaging
6342 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6343 {
6344 uint64_t u64Val = 0;
6345 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6346 if (pMixedCtx->cr3 != u64Val)
6347 {
6348 CPUMSetGuestCR3(pVCpu, u64Val);
6349 if (VMMRZCallRing3IsEnabled(pVCpu))
6350 {
6351 PGMUpdateCR3(pVCpu, u64Val);
6352 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6353 }
6354 else
6355 {
6356 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6357 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6358 }
6359 }
6360
6361 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6362 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6363 {
6364 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6365 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6366 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6367 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6368
6369 if (VMMRZCallRing3IsEnabled(pVCpu))
6370 {
6371 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6372 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6373 }
6374 else
6375 {
6376 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6377 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6378 }
6379 }
6380 }
6381
6382 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6383 }
6384
6385 /*
6386 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6387 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6388 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6389 *
6390 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6391 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6392 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6393 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6394 *
6395 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6396 */
6397 if (VMMRZCallRing3IsEnabled(pVCpu))
6398 {
6399 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6400 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6401
6402 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6403 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6404
6405 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6406 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6407 }
6408
6409 return rc;
6410}
6411
6412
6413/**
6414 * Reads a guest segment register from the current VMCS into the guest-CPU
6415 * context.
6416 *
6417 * @returns VBox status code.
6418 * @param pVCpu Pointer to the VMCPU.
6419 * @param idxSel Index of the selector in the VMCS.
6420 * @param idxLimit Index of the segment limit in the VMCS.
6421 * @param idxBase Index of the segment base in the VMCS.
6422 * @param idxAccess Index of the access rights of the segment in the VMCS.
6423 * @param pSelReg Pointer to the segment selector.
6424 *
6425 * @remarks No-long-jump zone!!!
6426 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6427 * macro as that takes care of whether to read from the VMCS cache or
6428 * not.
6429 */
6430DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6431 PCPUMSELREG pSelReg)
6432{
6433 NOREF(pVCpu);
6434
6435 uint32_t u32Val = 0;
6436 int rc = VMXReadVmcs32(idxSel, &u32Val);
6437 AssertRCReturn(rc, rc);
6438 pSelReg->Sel = (uint16_t)u32Val;
6439 pSelReg->ValidSel = (uint16_t)u32Val;
6440 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6441
6442 rc = VMXReadVmcs32(idxLimit, &u32Val);
6443 AssertRCReturn(rc, rc);
6444 pSelReg->u32Limit = u32Val;
6445
6446 uint64_t u64Val = 0;
6447 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6448 AssertRCReturn(rc, rc);
6449 pSelReg->u64Base = u64Val;
6450
6451 rc = VMXReadVmcs32(idxAccess, &u32Val);
6452 AssertRCReturn(rc, rc);
6453 pSelReg->Attr.u = u32Val;
6454
6455 /*
6456 * If VT-x marks the segment as unusable, most other bits remain undefined:
6457 * - For CS the L, D and G bits have meaning.
6458 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6459 * - For the remaining data segments no bits are defined.
6460 *
6461 * The present bit and the unusable bit has been observed to be set at the
6462 * same time (the selector was supposed to be invalid as we started executing
6463 * a V8086 interrupt in ring-0).
6464 *
6465 * What should be important for the rest of the VBox code, is that the P bit is
6466 * cleared. Some of the other VBox code recognizes the unusable bit, but
6467 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6468 * safe side here, we'll strip off P and other bits we don't care about. If
6469 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6470 *
6471 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6472 */
6473 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6474 {
6475 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6476
6477 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6478 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6479 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6480
6481 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6482#ifdef DEBUG_bird
6483 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6484 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6485 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6486#endif
6487 }
6488 return VINF_SUCCESS;
6489}
6490
6491
6492#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6493# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6494 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6495 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6496#else
6497# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6498 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6499 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6500#endif
6501
6502
6503/**
6504 * Saves the guest segment registers from the current VMCS into the guest-CPU
6505 * context.
6506 *
6507 * @returns VBox status code.
6508 * @param pVCpu Pointer to the VMCPU.
6509 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6510 * out-of-sync. Make sure to update the required fields
6511 * before using them.
6512 *
6513 * @remarks No-long-jump zone!!!
6514 */
6515static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6516{
6517 /* Guest segment registers. */
6518 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6519 {
6520 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6521 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6522 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6523 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6524 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6525 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6526 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6527
6528 /* Restore segment attributes for real-on-v86 mode hack. */
6529 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6530 {
6531 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6532 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6533 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6534 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6535 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6536 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6537 }
6538 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6539 }
6540
6541 return VINF_SUCCESS;
6542}
6543
6544
6545/**
6546 * Saves the guest descriptor table registers and task register from the current
6547 * VMCS into the guest-CPU context.
6548 *
6549 * @returns VBox status code.
6550 * @param pVCpu Pointer to the VMCPU.
6551 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6552 * out-of-sync. Make sure to update the required fields
6553 * before using them.
6554 *
6555 * @remarks No-long-jump zone!!!
6556 */
6557static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6558{
6559 int rc = VINF_SUCCESS;
6560
6561 /* Guest LDTR. */
6562 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6563 {
6564 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6565 AssertRCReturn(rc, rc);
6566 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6567 }
6568
6569 /* Guest GDTR. */
6570 uint64_t u64Val = 0;
6571 uint32_t u32Val = 0;
6572 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6573 {
6574 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6575 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6576 pMixedCtx->gdtr.pGdt = u64Val;
6577 pMixedCtx->gdtr.cbGdt = u32Val;
6578 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6579 }
6580
6581 /* Guest IDTR. */
6582 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6583 {
6584 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6585 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6586 pMixedCtx->idtr.pIdt = u64Val;
6587 pMixedCtx->idtr.cbIdt = u32Val;
6588 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6589 }
6590
6591 /* Guest TR. */
6592 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6593 {
6594 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6595 AssertRCReturn(rc, rc);
6596
6597 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6598 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6599 {
6600 rc = VMXLOCAL_READ_SEG(TR, tr);
6601 AssertRCReturn(rc, rc);
6602 }
6603 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6604 }
6605 return rc;
6606}
6607
6608#undef VMXLOCAL_READ_SEG
6609
6610
6611/**
6612 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6613 * context.
6614 *
6615 * @returns VBox status code.
6616 * @param pVCpu Pointer to the VMCPU.
6617 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6618 * out-of-sync. Make sure to update the required fields
6619 * before using them.
6620 *
6621 * @remarks No-long-jump zone!!!
6622 */
6623static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6624{
6625 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6626 {
6627 if (!pVCpu->hm.s.fUsingHyperDR7)
6628 {
6629 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6630 uint32_t u32Val;
6631 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6632 pMixedCtx->dr[7] = u32Val;
6633 }
6634
6635 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6636 }
6637 return VINF_SUCCESS;
6638}
6639
6640
6641/**
6642 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6643 *
6644 * @returns VBox status code.
6645 * @param pVCpu Pointer to the VMCPU.
6646 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6647 * out-of-sync. Make sure to update the required fields
6648 * before using them.
6649 *
6650 * @remarks No-long-jump zone!!!
6651 */
6652static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6653{
6654 NOREF(pMixedCtx);
6655
6656 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6657 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6658 return VINF_SUCCESS;
6659}
6660
6661
6662/**
6663 * Saves the entire guest state from the currently active VMCS into the
6664 * guest-CPU context. This essentially VMREADs all guest-data.
6665 *
6666 * @returns VBox status code.
6667 * @param pVCpu Pointer to the VMCPU.
6668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6669 * out-of-sync. Make sure to update the required fields
6670 * before using them.
6671 */
6672static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6673{
6674 Assert(pVCpu);
6675 Assert(pMixedCtx);
6676
6677 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6678 return VINF_SUCCESS;
6679
6680 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6681 again on the ring-3 callback path, there is no real need to. */
6682 if (VMMRZCallRing3IsEnabled(pVCpu))
6683 VMMR0LogFlushDisable(pVCpu);
6684 else
6685 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6686 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6687
6688 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6689 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6690
6691 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6692 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6693
6694 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6695 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6696
6697 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6698 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6699
6700 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6701 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6702
6703 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6704 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6705
6706 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6707 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6708
6709 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6710 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6711
6712 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6713 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6714
6715 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6716 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6717
6718 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6719 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6720
6721 if (VMMRZCallRing3IsEnabled(pVCpu))
6722 VMMR0LogFlushEnable(pVCpu);
6723
6724 return rc;
6725}
6726
6727
6728/**
6729 * Check per-VM and per-VCPU force flag actions that require us to go back to
6730 * ring-3 for one reason or another.
6731 *
6732 * @returns VBox status code (information status code included).
6733 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6734 * ring-3.
6735 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6736 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6737 * interrupts)
6738 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6739 * all EMTs to be in ring-3.
6740 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6741 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6742 * to the EM loop.
6743 *
6744 * @param pVM Pointer to the VM.
6745 * @param pVCpu Pointer to the VMCPU.
6746 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6747 * out-of-sync. Make sure to update the required fields
6748 * before using them.
6749 */
6750static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6751{
6752 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6753
6754 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6755 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6756 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6757 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6758 {
6759 /* We need the control registers now, make sure the guest-CPU context is updated. */
6760 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6761 AssertRCReturn(rc3, rc3);
6762
6763 /* Pending HM CR3 sync. */
6764 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6765 {
6766 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6767 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6768 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6769 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6770 }
6771
6772 /* Pending HM PAE PDPEs. */
6773 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6774 {
6775 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6776 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6777 }
6778
6779 /* Pending PGM C3 sync. */
6780 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6781 {
6782 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6783 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6784 if (rc2 != VINF_SUCCESS)
6785 {
6786 AssertRC(rc2);
6787 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6788 return rc2;
6789 }
6790 }
6791
6792 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6793 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6794 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6795 {
6796 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6797 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6798 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6799 return rc2;
6800 }
6801
6802 /* Pending VM request packets, such as hardware interrupts. */
6803 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6804 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6805 {
6806 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6807 return VINF_EM_PENDING_REQUEST;
6808 }
6809
6810 /* Pending PGM pool flushes. */
6811 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6812 {
6813 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6814 return VINF_PGM_POOL_FLUSH_PENDING;
6815 }
6816
6817 /* Pending DMA requests. */
6818 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6819 {
6820 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6821 return VINF_EM_RAW_TO_R3;
6822 }
6823 }
6824
6825 return VINF_SUCCESS;
6826}
6827
6828
6829/**
6830 * Converts any TRPM trap into a pending HM event. This is typically used when
6831 * entering from ring-3 (not longjmp returns).
6832 *
6833 * @param pVCpu Pointer to the VMCPU.
6834 */
6835static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6836{
6837 Assert(TRPMHasTrap(pVCpu));
6838 Assert(!pVCpu->hm.s.Event.fPending);
6839
6840 uint8_t uVector;
6841 TRPMEVENT enmTrpmEvent;
6842 RTGCUINT uErrCode;
6843 RTGCUINTPTR GCPtrFaultAddress;
6844 uint8_t cbInstr;
6845
6846 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6847 AssertRC(rc);
6848
6849 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6850 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6851 if (enmTrpmEvent == TRPM_TRAP)
6852 {
6853 switch (uVector)
6854 {
6855 case X86_XCPT_NMI:
6856 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6857 break;
6858
6859 case X86_XCPT_BP:
6860 case X86_XCPT_OF:
6861 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6862 break;
6863
6864 case X86_XCPT_PF:
6865 case X86_XCPT_DF:
6866 case X86_XCPT_TS:
6867 case X86_XCPT_NP:
6868 case X86_XCPT_SS:
6869 case X86_XCPT_GP:
6870 case X86_XCPT_AC:
6871 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6872 /* no break! */
6873 default:
6874 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6875 break;
6876 }
6877 }
6878 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6879 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6880 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6881 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6882 else
6883 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6884
6885 rc = TRPMResetTrap(pVCpu);
6886 AssertRC(rc);
6887 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6888 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6889
6890 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6891 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6892}
6893
6894
6895/**
6896 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6897 * VT-x to execute any instruction.
6898 *
6899 * @param pvCpu Pointer to the VMCPU.
6900 */
6901static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6902{
6903 Assert(pVCpu->hm.s.Event.fPending);
6904
6905 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6906 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6907 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6908 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6909
6910 /* If a trap was already pending, we did something wrong! */
6911 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6912
6913 TRPMEVENT enmTrapType;
6914 switch (uVectorType)
6915 {
6916 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6917 enmTrapType = TRPM_HARDWARE_INT;
6918 break;
6919
6920 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6921 enmTrapType = TRPM_SOFTWARE_INT;
6922 break;
6923
6924 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6925 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6926 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6927 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6928 enmTrapType = TRPM_TRAP;
6929 break;
6930
6931 default:
6932 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6933 enmTrapType = TRPM_32BIT_HACK;
6934 break;
6935 }
6936
6937 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6938
6939 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6940 AssertRC(rc);
6941
6942 if (fErrorCodeValid)
6943 TRPMSetErrorCode(pVCpu, uErrorCode);
6944
6945 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6946 && uVector == X86_XCPT_PF)
6947 {
6948 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6949 }
6950 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6951 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6952 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6953 {
6954 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6955 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6956 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6957 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6958 }
6959 pVCpu->hm.s.Event.fPending = false;
6960}
6961
6962
6963/**
6964 * Does the necessary state syncing before returning to ring-3 for any reason
6965 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6966 *
6967 * @returns VBox status code.
6968 * @param pVM Pointer to the VM.
6969 * @param pVCpu Pointer to the VMCPU.
6970 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6971 * be out-of-sync. Make sure to update the required
6972 * fields before using them.
6973 * @param fSaveGuestState Whether to save the guest state or not.
6974 *
6975 * @remarks No-long-jmp zone!!!
6976 */
6977static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6978{
6979 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6980 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6981
6982 RTCPUID idCpu = RTMpCpuId();
6983 Log4Func(("HostCpuId=%u\n", idCpu));
6984
6985 /*
6986 * !!! IMPORTANT !!!
6987 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6988 */
6989
6990 /* Save the guest state if necessary. */
6991 if ( fSaveGuestState
6992 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6993 {
6994 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6995 AssertRCReturn(rc, rc);
6996 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6997 }
6998
6999 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7000 if (CPUMIsGuestFPUStateActive(pVCpu))
7001 {
7002 /* We shouldn't reload CR0 without saving it first. */
7003 if (!fSaveGuestState)
7004 {
7005 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7006 AssertRCReturn(rc, rc);
7007 }
7008 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7009 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7010 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7011 }
7012
7013 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7014#ifdef VBOX_STRICT
7015 if (CPUMIsHyperDebugStateActive(pVCpu))
7016 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7017#endif
7018 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7019 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7020 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7021 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7022
7023#if HC_ARCH_BITS == 64
7024 /* Restore host-state bits that VT-x only restores partially. */
7025 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7026 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7027 {
7028 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7029 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7030 }
7031 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7032#endif
7033
7034#if HC_ARCH_BITS == 64
7035 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7036 if ( pVM->hm.s.fAllow64BitGuests
7037 && pVCpu->hm.s.vmx.fLazyMsrs)
7038 {
7039 /* We shouldn't reload the guest MSRs without saving it first. */
7040 if (!fSaveGuestState)
7041 {
7042 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7043 AssertRCReturn(rc, rc);
7044 }
7045 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7046 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7047 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7048 }
7049#endif
7050
7051 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7052 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7053
7054 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7055 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7056 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7057 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7058 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7059 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7060 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7061 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7062
7063 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7064
7065 /** @todo This partially defeats the purpose of having preemption hooks.
7066 * The problem is, deregistering the hooks should be moved to a place that
7067 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7068 * context.
7069 */
7070 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7071 {
7072 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7073 AssertRCReturn(rc, rc);
7074
7075 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7076 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7077 }
7078 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7079 NOREF(idCpu);
7080
7081 return VINF_SUCCESS;
7082}
7083
7084
7085/**
7086 * Leaves the VT-x session.
7087 *
7088 * @returns VBox status code.
7089 * @param pVM Pointer to the VM.
7090 * @param pVCpu Pointer to the VMCPU.
7091 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7092 * out-of-sync. Make sure to update the required fields
7093 * before using them.
7094 *
7095 * @remarks No-long-jmp zone!!!
7096 */
7097DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7098{
7099 HM_DISABLE_PREEMPT_IF_NEEDED();
7100 HMVMX_ASSERT_CPU_SAFE();
7101 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7102 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7103
7104 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7105 and done this from the VMXR0ThreadCtxCallback(). */
7106 if (!pVCpu->hm.s.fLeaveDone)
7107 {
7108 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7109 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7110 pVCpu->hm.s.fLeaveDone = true;
7111 }
7112 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7113
7114 /*
7115 * !!! IMPORTANT !!!
7116 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7117 */
7118
7119 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7120 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7121 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7122 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7123 VMMR0ThreadCtxHooksDeregister(pVCpu);
7124
7125 /* Leave HM context. This takes care of local init (term). */
7126 int rc = HMR0LeaveCpu(pVCpu);
7127
7128 HM_RESTORE_PREEMPT_IF_NEEDED();
7129
7130 return rc;
7131}
7132
7133
7134/**
7135 * Does the necessary state syncing before doing a longjmp to ring-3.
7136 *
7137 * @returns VBox status code.
7138 * @param pVM Pointer to the VM.
7139 * @param pVCpu Pointer to the VMCPU.
7140 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7141 * out-of-sync. Make sure to update the required fields
7142 * before using them.
7143 *
7144 * @remarks No-long-jmp zone!!!
7145 */
7146DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7147{
7148 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7149}
7150
7151
7152/**
7153 * Take necessary actions before going back to ring-3.
7154 *
7155 * An action requires us to go back to ring-3. This function does the necessary
7156 * steps before we can safely return to ring-3. This is not the same as longjmps
7157 * to ring-3, this is voluntary and prepares the guest so it may continue
7158 * executing outside HM (recompiler/IEM).
7159 *
7160 * @returns VBox status code.
7161 * @param pVM Pointer to the VM.
7162 * @param pVCpu Pointer to the VMCPU.
7163 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7164 * out-of-sync. Make sure to update the required fields
7165 * before using them.
7166 * @param rcExit The reason for exiting to ring-3. Can be
7167 * VINF_VMM_UNKNOWN_RING3_CALL.
7168 */
7169static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7170{
7171 Assert(pVM);
7172 Assert(pVCpu);
7173 Assert(pMixedCtx);
7174 HMVMX_ASSERT_PREEMPT_SAFE();
7175
7176 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7177 {
7178 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7179 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7180 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7181 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7182 }
7183
7184 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7185 VMMRZCallRing3Disable(pVCpu);
7186 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7187
7188 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7189 if (pVCpu->hm.s.Event.fPending)
7190 {
7191 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7192 Assert(!pVCpu->hm.s.Event.fPending);
7193 }
7194
7195 /* Save guest state and restore host state bits. */
7196 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7197 AssertRCReturn(rc, rc);
7198 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7199
7200 /* Sync recompiler state. */
7201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7202 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7203 | CPUM_CHANGED_LDTR
7204 | CPUM_CHANGED_GDTR
7205 | CPUM_CHANGED_IDTR
7206 | CPUM_CHANGED_TR
7207 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7208 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7209 if ( pVM->hm.s.fNestedPaging
7210 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7211 {
7212 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7213 }
7214
7215 Assert(!pVCpu->hm.s.fClearTrapFlag);
7216
7217 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7218 if (rcExit != VINF_EM_RAW_INTERRUPT)
7219 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7220
7221 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7222
7223 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7224 VMMRZCallRing3RemoveNotification(pVCpu);
7225 VMMRZCallRing3Enable(pVCpu);
7226
7227 return rc;
7228}
7229
7230
7231/**
7232 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7233 * longjump to ring-3 and possibly get preempted.
7234 *
7235 * @returns VBox status code.
7236 * @param pVCpu Pointer to the VMCPU.
7237 * @param enmOperation The operation causing the ring-3 longjump.
7238 * @param pvUser Opaque pointer to the guest-CPU context. The data
7239 * may be out-of-sync. Make sure to update the required
7240 * fields before using them.
7241 */
7242DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7243{
7244 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7245 {
7246 /*
7247 * !!! IMPORTANT !!!
7248 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7249 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7250 */
7251 VMMRZCallRing3RemoveNotification(pVCpu);
7252 VMMRZCallRing3Disable(pVCpu);
7253 HM_DISABLE_PREEMPT_IF_NEEDED();
7254
7255 PVM pVM = pVCpu->CTX_SUFF(pVM);
7256 if (CPUMIsGuestFPUStateActive(pVCpu))
7257 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7258
7259 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7260
7261#if HC_ARCH_BITS == 64
7262 /* Restore host-state bits that VT-x only restores partially. */
7263 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7264 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7265 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7266 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7267
7268 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7269 if ( pVM->hm.s.fAllow64BitGuests
7270 && pVCpu->hm.s.vmx.fLazyMsrs)
7271 {
7272 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7273 }
7274#endif
7275 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7276 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7277 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7278 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7279 {
7280 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7281 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7282 }
7283
7284 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7285 VMMR0ThreadCtxHooksDeregister(pVCpu);
7286
7287 HMR0LeaveCpu(pVCpu);
7288 HM_RESTORE_PREEMPT_IF_NEEDED();
7289 return VINF_SUCCESS;
7290 }
7291
7292 Assert(pVCpu);
7293 Assert(pvUser);
7294 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7295 HMVMX_ASSERT_PREEMPT_SAFE();
7296
7297 VMMRZCallRing3Disable(pVCpu);
7298 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7299
7300 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7301 enmOperation));
7302
7303 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7304 AssertRCReturn(rc, rc);
7305
7306 VMMRZCallRing3Enable(pVCpu);
7307 return VINF_SUCCESS;
7308}
7309
7310
7311/**
7312 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7313 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7314 *
7315 * @param pVCpu Pointer to the VMCPU.
7316 */
7317DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7318{
7319 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7320 {
7321 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7322 {
7323 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7324 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7325 AssertRC(rc);
7326 Log4(("Setup interrupt-window exiting\n"));
7327 }
7328 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7329}
7330
7331
7332/**
7333 * Clears the interrupt-window exiting control in the VMCS.
7334 *
7335 * @param pVCpu Pointer to the VMCPU.
7336 */
7337DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7338{
7339 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7340 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7342 AssertRC(rc);
7343 Log4(("Cleared interrupt-window exiting\n"));
7344}
7345
7346
7347/**
7348 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7349 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7350 *
7351 * @param pVCpu Pointer to the VMCPU.
7352 */
7353DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7354{
7355 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7356 {
7357 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7358 {
7359 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7360 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7361 AssertRC(rc);
7362 Log4(("Setup NMI-window exiting\n"));
7363 }
7364 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7365}
7366
7367
7368/**
7369 * Clears the NMI-window exiting control in the VMCS.
7370 *
7371 * @param pVCpu Pointer to the VMCPU.
7372 */
7373DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7374{
7375 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7376 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7377 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7378 AssertRC(rc);
7379 Log4(("Cleared NMI-window exiting\n"));
7380}
7381
7382
7383/**
7384 * Evaluates the event to be delivered to the guest and sets it as the pending
7385 * event.
7386 *
7387 * @param pVCpu Pointer to the VMCPU.
7388 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7389 * out-of-sync. Make sure to update the required fields
7390 * before using them.
7391 */
7392static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7393{
7394 Assert(!pVCpu->hm.s.Event.fPending);
7395
7396 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7397 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7398 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7399 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7400 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7401
7402 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7403 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7404 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7405 Assert(!TRPMHasTrap(pVCpu));
7406
7407 /*
7408 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7409 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7410 */
7411 /** @todo SMI. SMIs take priority over NMIs. */
7412 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7413 {
7414 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7415 if ( !fBlockNmi
7416 && !fBlockSti
7417 && !fBlockMovSS)
7418 {
7419 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7420 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7421 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7422
7423 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7424 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7425 }
7426 else
7427 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7428 }
7429 /*
7430 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7431 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7432 */
7433 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7434 && !pVCpu->hm.s.fSingleInstruction)
7435 {
7436 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7437 AssertRC(rc);
7438 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7439 if ( !fBlockInt
7440 && !fBlockSti
7441 && !fBlockMovSS)
7442 {
7443 uint8_t u8Interrupt;
7444 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7445 if (RT_SUCCESS(rc))
7446 {
7447 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7448 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7449 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7450
7451 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7452 }
7453 else
7454 {
7455 /** @todo Does this actually happen? If not turn it into an assertion. */
7456 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7457 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7458 }
7459 }
7460 else
7461 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7462 }
7463}
7464
7465
7466/**
7467 * Sets a pending-debug exception to be delivered to the guest if the guest is
7468 * single-stepping.
7469 *
7470 * @param pVCpu Pointer to the VMCPU.
7471 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7472 * out-of-sync. Make sure to update the required fields
7473 * before using them.
7474 */
7475DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7476{
7477 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7478 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7479 {
7480 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7481 AssertRC(rc);
7482 }
7483}
7484
7485
7486/**
7487 * Injects any pending events into the guest if the guest is in a state to
7488 * receive them.
7489 *
7490 * @returns VBox status code (informational status codes included).
7491 * @param pVCpu Pointer to the VMCPU.
7492 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7493 * out-of-sync. Make sure to update the required fields
7494 * before using them.
7495 */
7496static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7497{
7498 HMVMX_ASSERT_PREEMPT_SAFE();
7499 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7500
7501 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7502 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7503 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7504 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7505
7506 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7507 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7508 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7509 Assert(!TRPMHasTrap(pVCpu));
7510
7511 int rc = VINF_SUCCESS;
7512 if (pVCpu->hm.s.Event.fPending)
7513 {
7514 /*
7515 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7516 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7517 * ended up enabling interrupts outside VT-x.
7518 */
7519 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7520 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7521 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7522 {
7523 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7524 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7525 }
7526
7527#ifdef VBOX_STRICT
7528 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7529 {
7530 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7531 Assert(!fBlockInt);
7532 Assert(!fBlockSti);
7533 Assert(!fBlockMovSS);
7534 }
7535 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7536 {
7537 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7538 Assert(!fBlockSti);
7539 Assert(!fBlockMovSS);
7540 Assert(!fBlockNmi);
7541 }
7542#endif
7543 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7544 (uint8_t)uIntType));
7545 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7546 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7547 AssertRCReturn(rc, rc);
7548
7549 /* Update the interruptibility-state as it could have been changed by
7550 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7551 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7552 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7553
7554#ifdef VBOX_WITH_STATISTICS
7555 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7556 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7557 else
7558 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7559#endif
7560 }
7561
7562 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7563 if ( fBlockSti
7564 || fBlockMovSS)
7565 {
7566 if ( !pVCpu->hm.s.fSingleInstruction
7567 && !DBGFIsStepping(pVCpu))
7568 {
7569 /*
7570 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7571 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7572 * See Intel spec. 27.3.4 "Saving Non-Register State".
7573 */
7574 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7575 AssertRCReturn(rc2, rc2);
7576 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7577 }
7578 else if (pMixedCtx->eflags.Bits.u1TF)
7579 {
7580 /*
7581 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7582 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7583 */
7584 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7585 uIntrState = 0;
7586 }
7587 }
7588
7589 /*
7590 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7591 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7592 */
7593 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7594 AssertRC(rc2);
7595
7596 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7597 NOREF(fBlockMovSS); NOREF(fBlockSti);
7598 return rc;
7599}
7600
7601
7602/**
7603 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7604 *
7605 * @param pVCpu Pointer to the VMCPU.
7606 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7607 * out-of-sync. Make sure to update the required fields
7608 * before using them.
7609 */
7610DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7611{
7612 NOREF(pMixedCtx);
7613 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7614 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7615}
7616
7617
7618/**
7619 * Injects a double-fault (#DF) exception into the VM.
7620 *
7621 * @returns VBox status code (informational status code included).
7622 * @param pVCpu Pointer to the VMCPU.
7623 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7624 * out-of-sync. Make sure to update the required fields
7625 * before using them.
7626 */
7627DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7628{
7629 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7630 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7631 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7632 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7633 puIntrState);
7634}
7635
7636
7637/**
7638 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7639 *
7640 * @param pVCpu Pointer to the VMCPU.
7641 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7642 * out-of-sync. Make sure to update the required fields
7643 * before using them.
7644 */
7645DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7646{
7647 NOREF(pMixedCtx);
7648 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7649 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7650 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7651}
7652
7653
7654/**
7655 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7656 *
7657 * @param pVCpu Pointer to the VMCPU.
7658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7659 * out-of-sync. Make sure to update the required fields
7660 * before using them.
7661 * @param cbInstr The value of RIP that is to be pushed on the guest
7662 * stack.
7663 */
7664DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7665{
7666 NOREF(pMixedCtx);
7667 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7668 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7669 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7670}
7671
7672
7673/**
7674 * Injects a general-protection (#GP) fault into the VM.
7675 *
7676 * @returns VBox status code (informational status code included).
7677 * @param pVCpu Pointer to the VMCPU.
7678 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7679 * out-of-sync. Make sure to update the required fields
7680 * before using them.
7681 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7682 * mode, i.e. in real-mode it's not valid).
7683 * @param u32ErrorCode The error code associated with the #GP.
7684 */
7685DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7686 uint32_t *puIntrState)
7687{
7688 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7689 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7690 if (fErrorCodeValid)
7691 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7692 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7693 puIntrState);
7694}
7695
7696
7697/**
7698 * Sets a general-protection (#GP) exception as pending-for-injection into the
7699 * VM.
7700 *
7701 * @param pVCpu Pointer to the VMCPU.
7702 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7703 * out-of-sync. Make sure to update the required fields
7704 * before using them.
7705 * @param u32ErrorCode The error code associated with the #GP.
7706 */
7707DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7708{
7709 NOREF(pMixedCtx);
7710 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7711 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7712 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7713 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7714}
7715
7716
7717/**
7718 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7719 *
7720 * @param pVCpu Pointer to the VMCPU.
7721 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7722 * out-of-sync. Make sure to update the required fields
7723 * before using them.
7724 * @param uVector The software interrupt vector number.
7725 * @param cbInstr The value of RIP that is to be pushed on the guest
7726 * stack.
7727 */
7728DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7729{
7730 NOREF(pMixedCtx);
7731 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7732 if ( uVector == X86_XCPT_BP
7733 || uVector == X86_XCPT_OF)
7734 {
7735 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7736 }
7737 else
7738 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7739 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7740}
7741
7742
7743/**
7744 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7745 * stack.
7746 *
7747 * @returns VBox status code (information status code included).
7748 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7749 * @param pVM Pointer to the VM.
7750 * @param pMixedCtx Pointer to the guest-CPU context.
7751 * @param uValue The value to push to the guest stack.
7752 */
7753DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7754{
7755 /*
7756 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7757 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7758 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7759 */
7760 if (pMixedCtx->sp == 1)
7761 return VINF_EM_RESET;
7762 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7763 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7764 AssertRCReturn(rc, rc);
7765 return rc;
7766}
7767
7768
7769/**
7770 * Injects an event into the guest upon VM-entry by updating the relevant fields
7771 * in the VM-entry area in the VMCS.
7772 *
7773 * @returns VBox status code (informational error codes included).
7774 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7775 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7776 *
7777 * @param pVCpu Pointer to the VMCPU.
7778 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7779 * be out-of-sync. Make sure to update the required
7780 * fields before using them.
7781 * @param u64IntInfo The VM-entry interruption-information field.
7782 * @param cbInstr The VM-entry instruction length in bytes (for
7783 * software interrupts, exceptions and privileged
7784 * software exceptions).
7785 * @param u32ErrCode The VM-entry exception error code.
7786 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7787 * @param puIntrState Pointer to the current guest interruptibility-state.
7788 * This interruptibility-state will be updated if
7789 * necessary. This cannot not be NULL.
7790 *
7791 * @remarks Requires CR0!
7792 * @remarks No-long-jump zone!!!
7793 */
7794static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7795 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7796{
7797 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7798 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7799 Assert(puIntrState);
7800 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7801
7802 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7803 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7804
7805#ifdef VBOX_STRICT
7806 /* Validate the error-code-valid bit for hardware exceptions. */
7807 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7808 {
7809 switch (uVector)
7810 {
7811 case X86_XCPT_PF:
7812 case X86_XCPT_DF:
7813 case X86_XCPT_TS:
7814 case X86_XCPT_NP:
7815 case X86_XCPT_SS:
7816 case X86_XCPT_GP:
7817 case X86_XCPT_AC:
7818 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7819 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7820 /* fallthru */
7821 default:
7822 break;
7823 }
7824 }
7825#endif
7826
7827 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7828 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7829 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7830
7831 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7832
7833 /* We require CR0 to check if the guest is in real-mode. */
7834 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7835 AssertRCReturn(rc, rc);
7836
7837 /*
7838 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7839 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7840 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7841 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7842 */
7843 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7844 {
7845 PVM pVM = pVCpu->CTX_SUFF(pVM);
7846 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7847 {
7848 Assert(PDMVmmDevHeapIsEnabled(pVM));
7849 Assert(pVM->hm.s.vmx.pRealModeTSS);
7850
7851 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7852 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7853 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7854 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7855 AssertRCReturn(rc, rc);
7856 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7857
7858 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7859 size_t const cbIdtEntry = sizeof(X86IDTR16);
7860 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7861 {
7862 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7863 if (uVector == X86_XCPT_DF)
7864 return VINF_EM_RESET;
7865 else if (uVector == X86_XCPT_GP)
7866 {
7867 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7868 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7869 }
7870
7871 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7872 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7873 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7874 }
7875
7876 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7877 uint16_t uGuestIp = pMixedCtx->ip;
7878 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7879 {
7880 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7881 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7882 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7883 }
7884 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7885 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7886
7887 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7888 X86IDTR16 IdtEntry;
7889 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7890 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7891 AssertRCReturn(rc, rc);
7892
7893 /* Construct the stack frame for the interrupt/exception handler. */
7894 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7895 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7896 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7897 AssertRCReturn(rc, rc);
7898
7899 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7900 if (rc == VINF_SUCCESS)
7901 {
7902 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7903 pMixedCtx->rip = IdtEntry.offSel;
7904 pMixedCtx->cs.Sel = IdtEntry.uSel;
7905 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7906 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7907 && uVector == X86_XCPT_PF)
7908 {
7909 pMixedCtx->cr2 = GCPtrFaultAddress;
7910 }
7911
7912 /* If any other guest-state bits are changed here, make sure to update
7913 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7914 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7915 | HM_CHANGED_GUEST_RIP
7916 | HM_CHANGED_GUEST_RFLAGS
7917 | HM_CHANGED_GUEST_RSP);
7918
7919 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7920 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7921 {
7922 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7923 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7924 Log4(("Clearing inhibition due to STI.\n"));
7925 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7926 }
7927 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7928
7929 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7930 it, if we are returning to ring-3 before executing guest code. */
7931 pVCpu->hm.s.Event.fPending = false;
7932 }
7933 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7934 return rc;
7935 }
7936 else
7937 {
7938 /*
7939 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7940 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7941 */
7942 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7943 }
7944 }
7945
7946 /* Validate. */
7947 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7948 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7949 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7950
7951 /* Inject. */
7952 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7953 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7954 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7955 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7956
7957 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7958 && uVector == X86_XCPT_PF)
7959 {
7960 pMixedCtx->cr2 = GCPtrFaultAddress;
7961 }
7962
7963 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7964 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7965
7966 AssertRCReturn(rc, rc);
7967 return rc;
7968}
7969
7970
7971/**
7972 * Clears the interrupt-window exiting control in the VMCS and if necessary
7973 * clears the current event in the VMCS as well.
7974 *
7975 * @returns VBox status code.
7976 * @param pVCpu Pointer to the VMCPU.
7977 *
7978 * @remarks Use this function only to clear events that have not yet been
7979 * delivered to the guest but are injected in the VMCS!
7980 * @remarks No-long-jump zone!!!
7981 */
7982static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7983{
7984 int rc;
7985 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7986
7987 /* Clear interrupt-window exiting control. */
7988 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7989 {
7990 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7991 Assert(!pVCpu->hm.s.Event.fPending);
7992 }
7993
7994 /* Clear NMI-window exiting control. */
7995 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7996 {
7997 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7998 Assert(!pVCpu->hm.s.Event.fPending);
7999 }
8000
8001 if (!pVCpu->hm.s.Event.fPending)
8002 return;
8003
8004#ifdef VBOX_STRICT
8005 uint32_t u32EntryInfo;
8006 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8007 AssertRC(rc);
8008 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8009#endif
8010
8011 /* Clear the entry-interruption field (including the valid bit). */
8012 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8013 AssertRC(rc);
8014
8015 /* Clear the pending debug exception field. */
8016 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8017 AssertRC(rc);
8018
8019 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8020 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8021}
8022
8023
8024/**
8025 * Enters the VT-x session.
8026 *
8027 * @returns VBox status code.
8028 * @param pVM Pointer to the VM.
8029 * @param pVCpu Pointer to the VMCPU.
8030 * @param pCpu Pointer to the CPU info struct.
8031 */
8032VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8033{
8034 AssertPtr(pVM);
8035 AssertPtr(pVCpu);
8036 Assert(pVM->hm.s.vmx.fSupported);
8037 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8038 NOREF(pCpu); NOREF(pVM);
8039
8040 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8041 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8042
8043#ifdef VBOX_STRICT
8044 /* Make sure we're in VMX root mode. */
8045 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8046 if (!(u32HostCR4 & X86_CR4_VMXE))
8047 {
8048 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8049 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8050 }
8051#endif
8052
8053 /*
8054 * Load the VCPU's VMCS as the current (and active) one.
8055 */
8056 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8057 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8058 if (RT_FAILURE(rc))
8059 return rc;
8060
8061 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8062 pVCpu->hm.s.fLeaveDone = false;
8063 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8064
8065 return VINF_SUCCESS;
8066}
8067
8068
8069/**
8070 * The thread-context callback (only on platforms which support it).
8071 *
8072 * @param enmEvent The thread-context event.
8073 * @param pVCpu Pointer to the VMCPU.
8074 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8075 * @thread EMT(pVCpu)
8076 */
8077VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8078{
8079 NOREF(fGlobalInit);
8080
8081 switch (enmEvent)
8082 {
8083 case RTTHREADCTXEVENT_PREEMPTING:
8084 {
8085 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8086 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8087 VMCPU_ASSERT_EMT(pVCpu);
8088
8089 PVM pVM = pVCpu->CTX_SUFF(pVM);
8090 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8091
8092 /* No longjmps (logger flushes, locks) in this fragile context. */
8093 VMMRZCallRing3Disable(pVCpu);
8094 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8095
8096 /*
8097 * Restore host-state (FPU, debug etc.)
8098 */
8099 if (!pVCpu->hm.s.fLeaveDone)
8100 {
8101 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8102 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8103 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8104 pVCpu->hm.s.fLeaveDone = true;
8105 }
8106
8107 /* Leave HM context, takes care of local init (term). */
8108 int rc = HMR0LeaveCpu(pVCpu);
8109 AssertRC(rc); NOREF(rc);
8110
8111 /* Restore longjmp state. */
8112 VMMRZCallRing3Enable(pVCpu);
8113 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8114 break;
8115 }
8116
8117 case RTTHREADCTXEVENT_RESUMED:
8118 {
8119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8120 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8121 VMCPU_ASSERT_EMT(pVCpu);
8122
8123 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8124 VMMRZCallRing3Disable(pVCpu);
8125 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8126
8127 /* Initialize the bare minimum state required for HM. This takes care of
8128 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8129 int rc = HMR0EnterCpu(pVCpu);
8130 AssertRC(rc);
8131 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8132
8133 /* Load the active VMCS as the current one. */
8134 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8135 {
8136 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8137 AssertRC(rc); NOREF(rc);
8138 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8139 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8140 }
8141 pVCpu->hm.s.fLeaveDone = false;
8142
8143 /* Restore longjmp state. */
8144 VMMRZCallRing3Enable(pVCpu);
8145 break;
8146 }
8147
8148 default:
8149 break;
8150 }
8151}
8152
8153
8154/**
8155 * Saves the host state in the VMCS host-state.
8156 * Sets up the VM-exit MSR-load area.
8157 *
8158 * The CPU state will be loaded from these fields on every successful VM-exit.
8159 *
8160 * @returns VBox status code.
8161 * @param pVM Pointer to the VM.
8162 * @param pVCpu Pointer to the VMCPU.
8163 *
8164 * @remarks No-long-jump zone!!!
8165 */
8166static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8167{
8168 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8169
8170 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8171 return VINF_SUCCESS;
8172
8173 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8174 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8175
8176 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8177 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8178
8179 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8180 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8181
8182 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8183 return rc;
8184}
8185
8186
8187/**
8188 * Saves the host state in the VMCS host-state.
8189 *
8190 * @returns VBox status code.
8191 * @param pVM Pointer to the VM.
8192 * @param pVCpu Pointer to the VMCPU.
8193 *
8194 * @remarks No-long-jump zone!!!
8195 */
8196VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8197{
8198 AssertPtr(pVM);
8199 AssertPtr(pVCpu);
8200
8201 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8202
8203 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8204 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8205 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8206 return hmR0VmxSaveHostState(pVM, pVCpu);
8207}
8208
8209
8210/**
8211 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8212 * loaded from these fields on every successful VM-entry.
8213 *
8214 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8215 * Sets up the VM-entry controls.
8216 * Sets up the appropriate VMX non-root function to execute guest code based on
8217 * the guest CPU mode.
8218 *
8219 * @returns VBox status code.
8220 * @param pVM Pointer to the VM.
8221 * @param pVCpu Pointer to the VMCPU.
8222 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8223 * out-of-sync. Make sure to update the required fields
8224 * before using them.
8225 *
8226 * @remarks No-long-jump zone!!!
8227 */
8228static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8229{
8230 AssertPtr(pVM);
8231 AssertPtr(pVCpu);
8232 AssertPtr(pMixedCtx);
8233 HMVMX_ASSERT_PREEMPT_SAFE();
8234
8235#ifdef LOG_ENABLED
8236 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8237 * probably not initialized yet? Anyway this will do for now.
8238 *
8239 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8240 * interface and disable ring-3 calls when thread-context hooks are not
8241 * available. */
8242 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8243 VMMR0LogFlushDisable(pVCpu);
8244#endif
8245
8246 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8247
8248 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8249
8250 /* Determine real-on-v86 mode. */
8251 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8252 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8253 && CPUMIsGuestInRealModeEx(pMixedCtx))
8254 {
8255 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8256 }
8257
8258 /*
8259 * Load the guest-state into the VMCS.
8260 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8261 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8262 */
8263 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8264 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8265
8266 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8267 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8268 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8269
8270 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8271 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8272 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8273
8274 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8275 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8276
8277 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8278 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8279
8280 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8281 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8282 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8283
8284 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8285 determine we don't have to swap EFER after all. */
8286 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8287 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8288
8289 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8290 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8291
8292 /*
8293 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8294 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8295 */
8296 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8297 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8298
8299 /* Clear any unused and reserved bits. */
8300 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8301
8302#ifdef LOG_ENABLED
8303 /* Only reenable log-flushing if the caller has it enabled. */
8304 if (!fCallerDisabledLogFlush)
8305 VMMR0LogFlushEnable(pVCpu);
8306#endif
8307
8308 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8309 return rc;
8310}
8311
8312
8313/**
8314 * Loads the state shared between the host and guest into the VMCS.
8315 *
8316 * @param pVM Pointer to the VM.
8317 * @param pVCpu Pointer to the VMCPU.
8318 * @param pCtx Pointer to the guest-CPU context.
8319 *
8320 * @remarks No-long-jump zone!!!
8321 */
8322static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8323{
8324 NOREF(pVM);
8325
8326 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8327 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8328
8329 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8330 {
8331 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8332 AssertRC(rc);
8333 }
8334
8335 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8336 {
8337 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8338 AssertRC(rc);
8339
8340 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8341 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8342 {
8343 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8344 AssertRC(rc);
8345 }
8346 }
8347
8348 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8349 {
8350#if HC_ARCH_BITS == 64
8351 if (pVM->hm.s.fAllow64BitGuests)
8352 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8353#endif
8354 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8355 }
8356
8357 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8358 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8359}
8360
8361
8362/**
8363 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8364 *
8365 * @param pVM Pointer to the VM.
8366 * @param pVCpu Pointer to the VMCPU.
8367 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8368 * out-of-sync. Make sure to update the required fields
8369 * before using them.
8370 */
8371DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8372{
8373 HMVMX_ASSERT_PREEMPT_SAFE();
8374
8375 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8376#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8377 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8378#endif
8379
8380 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8381 {
8382 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8383 AssertRC(rc);
8384 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8385 }
8386 else if (HMCPU_CF_VALUE(pVCpu))
8387 {
8388 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8389 AssertRC(rc);
8390 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8391 }
8392
8393 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8394 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8395 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8396 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8397}
8398
8399
8400/**
8401 * Does the preparations before executing guest code in VT-x.
8402 *
8403 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8404 * recompiler. We must be cautious what we do here regarding committing
8405 * guest-state information into the VMCS assuming we assuredly execute the
8406 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8407 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8408 * so that the recompiler can (and should) use them when it resumes guest
8409 * execution. Otherwise such operations must be done when we can no longer
8410 * exit to ring-3.
8411 *
8412 * @returns Strict VBox status code.
8413 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8414 * have been disabled.
8415 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8416 * double-fault into the guest.
8417 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8418 *
8419 * @param pVM Pointer to the VM.
8420 * @param pVCpu Pointer to the VMCPU.
8421 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8422 * out-of-sync. Make sure to update the required fields
8423 * before using them.
8424 * @param pVmxTransient Pointer to the VMX transient structure.
8425 */
8426static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8427{
8428 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8429
8430#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8431 PGMRZDynMapFlushAutoSet(pVCpu);
8432#endif
8433
8434 /* Check force flag actions that might require us to go back to ring-3. */
8435 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8436 if (rc != VINF_SUCCESS)
8437 return rc;
8438
8439#ifndef IEM_VERIFICATION_MODE_FULL
8440 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8441 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8442 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8443 {
8444 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8445 RTGCPHYS GCPhysApicBase;
8446 GCPhysApicBase = pMixedCtx->msrApicBase;
8447 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8448
8449 /* Unalias any existing mapping. */
8450 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8451 AssertRCReturn(rc, rc);
8452
8453 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8454 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8455 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8456 AssertRCReturn(rc, rc);
8457
8458 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8459 }
8460#endif /* !IEM_VERIFICATION_MODE_FULL */
8461
8462 if (TRPMHasTrap(pVCpu))
8463 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8464 else if (!pVCpu->hm.s.Event.fPending)
8465 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8466
8467 /*
8468 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8469 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8470 */
8471 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8472 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8473 {
8474 Assert(rc == VINF_EM_RESET);
8475 return rc;
8476 }
8477
8478 /*
8479 * Load the guest state bits, we can handle longjmps/getting preempted here.
8480 *
8481 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8482 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8483 * Hence, this needs to be done -after- injection of events.
8484 */
8485 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8486
8487 /*
8488 * No longjmps to ring-3 from this point on!!!
8489 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8490 * This also disables flushing of the R0-logger instance (if any).
8491 */
8492 VMMRZCallRing3Disable(pVCpu);
8493
8494 /*
8495 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8496 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8497 *
8498 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8499 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8500 *
8501 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8502 * executing guest code.
8503 */
8504 pVmxTransient->uEflags = ASMIntDisableFlags();
8505 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8506 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8507 {
8508 hmR0VmxClearEventVmcs(pVCpu);
8509 ASMSetFlags(pVmxTransient->uEflags);
8510 VMMRZCallRing3Enable(pVCpu);
8511 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8512 return VINF_EM_RAW_TO_R3;
8513 }
8514
8515 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8516 {
8517 hmR0VmxClearEventVmcs(pVCpu);
8518 ASMSetFlags(pVmxTransient->uEflags);
8519 VMMRZCallRing3Enable(pVCpu);
8520 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8521 return VINF_EM_RAW_INTERRUPT;
8522 }
8523
8524 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8525 pVCpu->hm.s.Event.fPending = false;
8526
8527 return VINF_SUCCESS;
8528}
8529
8530
8531/**
8532 * Prepares to run guest code in VT-x and we've committed to doing so. This
8533 * means there is no backing out to ring-3 or anywhere else at this
8534 * point.
8535 *
8536 * @param pVM Pointer to the VM.
8537 * @param pVCpu Pointer to the VMCPU.
8538 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8539 * out-of-sync. Make sure to update the required fields
8540 * before using them.
8541 * @param pVmxTransient Pointer to the VMX transient structure.
8542 *
8543 * @remarks Called with preemption disabled.
8544 * @remarks No-long-jump zone!!!
8545 */
8546static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8547{
8548 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8549 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8550 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8551
8552 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8553 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8554
8555#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8556 if (!CPUMIsGuestFPUStateActive(pVCpu))
8557 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8558 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8559#endif
8560
8561 if ( pVCpu->hm.s.fUseGuestFpu
8562 && !CPUMIsGuestFPUStateActive(pVCpu))
8563 {
8564 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8565 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8566 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8567 }
8568
8569 /*
8570 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8571 */
8572 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8573 && pVCpu->hm.s.vmx.cMsrs > 0)
8574 {
8575 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8576 }
8577
8578 /*
8579 * Load the host state bits as we may've been preempted (only happens when
8580 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8581 */
8582 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8583 {
8584 /* This ASSUMES that pfnStartVM has been set up already. */
8585 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8586 AssertRC(rc);
8587 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8588 }
8589 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8590
8591 /*
8592 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8593 */
8594 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8595 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8596 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8597
8598 /* Store status of the shared guest-host state at the time of VM-entry. */
8599#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8600 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8601 {
8602 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8603 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8604 }
8605 else
8606#endif
8607 {
8608 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8609 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8610 }
8611 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8612
8613 /*
8614 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8615 */
8616 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8617 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8618
8619 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8620 RTCPUID idCurrentCpu = pCpu->idCpu;
8621 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8622 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8623 {
8624 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8625 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8626 }
8627
8628 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8629 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8630 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8631 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8632
8633 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8634
8635 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8636 to start executing. */
8637
8638 /*
8639 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8640 */
8641 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8642 {
8643 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8644 {
8645 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8646 AssertRC(rc2);
8647 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8648 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8649 true /* fUpdateHostMsr */);
8650 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8651 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8652 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8653 }
8654 else
8655 {
8656 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8657 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8658 }
8659 }
8660
8661#ifdef VBOX_STRICT
8662 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8663 hmR0VmxCheckHostEferMsr(pVCpu);
8664 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8665#endif
8666#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8667 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8668 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8669 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8670#endif
8671}
8672
8673
8674/**
8675 * Performs some essential restoration of state after running guest code in
8676 * VT-x.
8677 *
8678 * @param pVM Pointer to the VM.
8679 * @param pVCpu Pointer to the VMCPU.
8680 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8681 * out-of-sync. Make sure to update the required fields
8682 * before using them.
8683 * @param pVmxTransient Pointer to the VMX transient structure.
8684 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8685 *
8686 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8687 *
8688 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8689 * unconditionally when it is safe to do so.
8690 */
8691static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8692{
8693 NOREF(pVM);
8694
8695 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8696
8697 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8698 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8699 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8700 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8701 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8702
8703 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8704 {
8705 /** @todo Find a way to fix hardcoding a guestimate. */
8706 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8707 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8708 }
8709
8710 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8711 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8712 Assert(!(ASMGetFlags() & X86_EFL_IF));
8713 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8714
8715#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8716 if (CPUMIsGuestFPUStateActive(pVCpu))
8717 {
8718 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8719 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8720 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8721 }
8722#endif
8723
8724#if HC_ARCH_BITS == 64
8725 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8726#endif
8727 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8728#ifdef VBOX_STRICT
8729 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8730#endif
8731 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8732 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8733
8734 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8735 uint32_t uExitReason;
8736 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8737 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8738 AssertRC(rc);
8739 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8740 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8741
8742 /* Update the VM-exit history array. */
8743 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8744
8745 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8746 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8747 {
8748 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8749 pVmxTransient->fVMEntryFailed));
8750 return;
8751 }
8752
8753 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8754 {
8755 /** @todo We can optimize this by only syncing with our force-flags when
8756 * really needed and keeping the VMCS state as it is for most
8757 * VM-exits. */
8758 /* Update the guest interruptibility-state from the VMCS. */
8759 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8760
8761#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8762 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8763 AssertRC(rc);
8764#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8765 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8766 AssertRC(rc);
8767#endif
8768
8769 /*
8770 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8771 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8772 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8773 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8774 */
8775 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8776 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8777 {
8778 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8779 AssertRC(rc);
8780 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8781 }
8782 }
8783}
8784
8785
8786/**
8787 * Runs the guest code using VT-x the normal way.
8788 *
8789 * @returns VBox status code.
8790 * @param pVM Pointer to the VM.
8791 * @param pVCpu Pointer to the VMCPU.
8792 * @param pCtx Pointer to the guest-CPU context.
8793 *
8794 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8795 */
8796static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8797{
8798 VMXTRANSIENT VmxTransient;
8799 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8800 int rc = VERR_INTERNAL_ERROR_5;
8801 uint32_t cLoops = 0;
8802
8803 for (;; cLoops++)
8804 {
8805 Assert(!HMR0SuspendPending());
8806 HMVMX_ASSERT_CPU_SAFE();
8807
8808 /* Preparatory work for running guest code, this may force us to return
8809 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8810 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8811 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8812 if (rc != VINF_SUCCESS)
8813 break;
8814
8815 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8816 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8817 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8818
8819 /* Restore any residual host-state and save any bits shared between host
8820 and guest into the guest-CPU state. Re-enables interrupts! */
8821 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8822
8823 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8824 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8825 {
8826 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8827 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8828 return rc;
8829 }
8830
8831 /* Handle the VM-exit. */
8832 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8833 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8834 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8835 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8836 HMVMX_START_EXIT_DISPATCH_PROF();
8837#ifdef HMVMX_USE_FUNCTION_TABLE
8838 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8839#else
8840 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8841#endif
8842 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8843 if (rc != VINF_SUCCESS)
8844 break;
8845 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8846 {
8847 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8848 rc = VINF_EM_RAW_INTERRUPT;
8849 break;
8850 }
8851 }
8852
8853 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8854 return rc;
8855}
8856
8857
8858/**
8859 * Single steps guest code using VT-x.
8860 *
8861 * @returns VBox status code.
8862 * @param pVM Pointer to the VM.
8863 * @param pVCpu Pointer to the VMCPU.
8864 * @param pCtx Pointer to the guest-CPU context.
8865 *
8866 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8867 */
8868static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8869{
8870 VMXTRANSIENT VmxTransient;
8871 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8872 int rc = VERR_INTERNAL_ERROR_5;
8873 uint32_t cLoops = 0;
8874 uint16_t uCsStart = pCtx->cs.Sel;
8875 uint64_t uRipStart = pCtx->rip;
8876
8877 for (;; cLoops++)
8878 {
8879 Assert(!HMR0SuspendPending());
8880 HMVMX_ASSERT_CPU_SAFE();
8881
8882 /* Preparatory work for running guest code, this may force us to return
8883 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8884 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8885 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8886 if (rc != VINF_SUCCESS)
8887 break;
8888
8889 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8890 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8891 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8892
8893 /* Restore any residual host-state and save any bits shared between host
8894 and guest into the guest-CPU state. Re-enables interrupts! */
8895 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8896
8897 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8898 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8899 {
8900 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8901 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8902 return rc;
8903 }
8904
8905 /* Handle the VM-exit. */
8906 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8907 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8908 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8909 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8910 HMVMX_START_EXIT_DISPATCH_PROF();
8911#ifdef HMVMX_USE_FUNCTION_TABLE
8912 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8913#else
8914 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8915#endif
8916 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8917 if (rc != VINF_SUCCESS)
8918 break;
8919 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8920 {
8921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8922 rc = VINF_EM_RAW_INTERRUPT;
8923 break;
8924 }
8925
8926 /*
8927 * Did the RIP change, if so, consider it a single step.
8928 * Otherwise, make sure one of the TFs gets set.
8929 */
8930 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8931 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8932 AssertRCReturn(rc2, rc2);
8933 if ( pCtx->rip != uRipStart
8934 || pCtx->cs.Sel != uCsStart)
8935 {
8936 rc = VINF_EM_DBG_STEPPED;
8937 break;
8938 }
8939 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8940 }
8941
8942 /*
8943 * Clear the X86_EFL_TF if necessary.
8944 */
8945 if (pVCpu->hm.s.fClearTrapFlag)
8946 {
8947 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8948 AssertRCReturn(rc2, rc2);
8949 pVCpu->hm.s.fClearTrapFlag = false;
8950 pCtx->eflags.Bits.u1TF = 0;
8951 }
8952 /** @todo there seems to be issues with the resume flag when the monitor trap
8953 * flag is pending without being used. Seen early in bios init when
8954 * accessing APIC page in protected mode. */
8955
8956 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8957 return rc;
8958}
8959
8960
8961/**
8962 * Runs the guest code using VT-x.
8963 *
8964 * @returns VBox status code.
8965 * @param pVM Pointer to the VM.
8966 * @param pVCpu Pointer to the VMCPU.
8967 * @param pCtx Pointer to the guest-CPU context.
8968 */
8969VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8970{
8971 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8972 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8973 HMVMX_ASSERT_PREEMPT_SAFE();
8974
8975 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8976
8977 int rc;
8978 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8979 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8980 else
8981 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8982
8983 if (rc == VERR_EM_INTERPRETER)
8984 rc = VINF_EM_RAW_EMULATE_INSTR;
8985 else if (rc == VINF_EM_RESET)
8986 rc = VINF_EM_TRIPLE_FAULT;
8987
8988 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8989 if (RT_FAILURE(rc2))
8990 {
8991 pVCpu->hm.s.u32HMError = rc;
8992 rc = rc2;
8993 }
8994 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8995 return rc;
8996}
8997
8998
8999#ifndef HMVMX_USE_FUNCTION_TABLE
9000DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9001{
9002#ifdef DEBUG_ramshankar
9003# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9004# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9005#endif
9006 int rc;
9007 switch (rcReason)
9008 {
9009 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9010 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9011 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9012 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9013 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043
9044 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9045 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9046 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9047 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9048 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9049 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9050 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9051 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9052 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9053
9054 case VMX_EXIT_VMCLEAR:
9055 case VMX_EXIT_VMLAUNCH:
9056 case VMX_EXIT_VMPTRLD:
9057 case VMX_EXIT_VMPTRST:
9058 case VMX_EXIT_VMREAD:
9059 case VMX_EXIT_VMRESUME:
9060 case VMX_EXIT_VMWRITE:
9061 case VMX_EXIT_VMXOFF:
9062 case VMX_EXIT_VMXON:
9063 case VMX_EXIT_INVEPT:
9064 case VMX_EXIT_INVVPID:
9065 case VMX_EXIT_VMFUNC:
9066 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9067 break;
9068 default:
9069 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9070 break;
9071 }
9072 return rc;
9073}
9074#endif
9075
9076#ifdef DEBUG
9077/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9078# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9079 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9080
9081# define HMVMX_ASSERT_PREEMPT_CPUID() \
9082 do \
9083 { \
9084 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9085 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9086 } while (0)
9087
9088# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9089 do { \
9090 AssertPtr(pVCpu); \
9091 AssertPtr(pMixedCtx); \
9092 AssertPtr(pVmxTransient); \
9093 Assert(pVmxTransient->fVMEntryFailed == false); \
9094 Assert(ASMIntAreEnabled()); \
9095 HMVMX_ASSERT_PREEMPT_SAFE(); \
9096 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9097 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9098 HMVMX_ASSERT_PREEMPT_SAFE(); \
9099 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9100 HMVMX_ASSERT_PREEMPT_CPUID(); \
9101 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9102 } while (0)
9103
9104# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9105 do { \
9106 Log4Func(("\n")); \
9107 } while (0)
9108#else /* Release builds */
9109# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9110 do { \
9111 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9112 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9113 } while (0)
9114# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9115#endif
9116
9117
9118/**
9119 * Advances the guest RIP after reading it from the VMCS.
9120 *
9121 * @returns VBox status code.
9122 * @param pVCpu Pointer to the VMCPU.
9123 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9124 * out-of-sync. Make sure to update the required fields
9125 * before using them.
9126 * @param pVmxTransient Pointer to the VMX transient structure.
9127 *
9128 * @remarks No-long-jump zone!!!
9129 */
9130DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9131{
9132 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9133 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9134 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9135 AssertRCReturn(rc, rc);
9136
9137 pMixedCtx->rip += pVmxTransient->cbInstr;
9138 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9139
9140 /*
9141 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9142 * pending debug exception field as it takes care of priority of events.
9143 *
9144 * See Intel spec. 32.2.1 "Debug Exceptions".
9145 */
9146 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9147
9148 return rc;
9149}
9150
9151
9152/**
9153 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9154 * and update error record fields accordingly.
9155 *
9156 * @return VMX_IGS_* return codes.
9157 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9158 * wrong with the guest state.
9159 *
9160 * @param pVM Pointer to the VM.
9161 * @param pVCpu Pointer to the VMCPU.
9162 * @param pCtx Pointer to the guest-CPU state.
9163 *
9164 * @remarks This function assumes our cache of the VMCS controls
9165 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9166 */
9167static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9168{
9169#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9170#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9171 uError = (err); \
9172 break; \
9173 } else do { } while (0)
9174
9175 int rc;
9176 uint32_t uError = VMX_IGS_ERROR;
9177 uint32_t u32Val;
9178 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9179
9180 do
9181 {
9182 /*
9183 * CR0.
9184 */
9185 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9186 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9187 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9188 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9189 if (fUnrestrictedGuest)
9190 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9191
9192 uint32_t u32GuestCR0;
9193 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9194 AssertRCBreak(rc);
9195 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9196 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9197 if ( !fUnrestrictedGuest
9198 && (u32GuestCR0 & X86_CR0_PG)
9199 && !(u32GuestCR0 & X86_CR0_PE))
9200 {
9201 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9202 }
9203
9204 /*
9205 * CR4.
9206 */
9207 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9208 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9209
9210 uint32_t u32GuestCR4;
9211 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9212 AssertRCBreak(rc);
9213 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9214 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9215
9216 /*
9217 * IA32_DEBUGCTL MSR.
9218 */
9219 uint64_t u64Val;
9220 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9221 AssertRCBreak(rc);
9222 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9223 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9224 {
9225 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9226 }
9227 uint64_t u64DebugCtlMsr = u64Val;
9228
9229#ifdef VBOX_STRICT
9230 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9231 AssertRCBreak(rc);
9232 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9233#endif
9234 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9235
9236 /*
9237 * RIP and RFLAGS.
9238 */
9239 uint32_t u32Eflags;
9240#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9241 if (HMVMX_IS_64BIT_HOST_MODE())
9242 {
9243 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9244 AssertRCBreak(rc);
9245 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9246 if ( !fLongModeGuest
9247 || !pCtx->cs.Attr.n.u1Long)
9248 {
9249 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9250 }
9251 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9252 * must be identical if the "IA-32e mode guest" VM-entry
9253 * control is 1 and CS.L is 1. No check applies if the
9254 * CPU supports 64 linear-address bits. */
9255
9256 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9257 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9258 AssertRCBreak(rc);
9259 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9260 VMX_IGS_RFLAGS_RESERVED);
9261 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9262 u32Eflags = u64Val;
9263 }
9264 else
9265#endif
9266 {
9267 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9268 AssertRCBreak(rc);
9269 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9270 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9271 }
9272
9273 if ( fLongModeGuest
9274 || ( fUnrestrictedGuest
9275 && !(u32GuestCR0 & X86_CR0_PE)))
9276 {
9277 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9278 }
9279
9280 uint32_t u32EntryInfo;
9281 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9282 AssertRCBreak(rc);
9283 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9284 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9285 {
9286 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9287 }
9288
9289 /*
9290 * 64-bit checks.
9291 */
9292#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9293 if (HMVMX_IS_64BIT_HOST_MODE())
9294 {
9295 if ( fLongModeGuest
9296 && !fUnrestrictedGuest)
9297 {
9298 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9299 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9300 }
9301
9302 if ( !fLongModeGuest
9303 && (u32GuestCR4 & X86_CR4_PCIDE))
9304 {
9305 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9306 }
9307
9308 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9309 * 51:32 beyond the processor's physical-address width are 0. */
9310
9311 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9312 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9313 {
9314 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9315 }
9316
9317 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9318 AssertRCBreak(rc);
9319 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9320
9321 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9322 AssertRCBreak(rc);
9323 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9324 }
9325#endif
9326
9327 /*
9328 * PERF_GLOBAL MSR.
9329 */
9330 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9331 {
9332 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9333 AssertRCBreak(rc);
9334 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9335 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9336 }
9337
9338 /*
9339 * PAT MSR.
9340 */
9341 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9342 {
9343 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9344 AssertRCBreak(rc);
9345 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9346 for (unsigned i = 0; i < 8; i++)
9347 {
9348 uint8_t u8Val = (u64Val & 0x7);
9349 if ( u8Val != 0 /* UC */
9350 || u8Val != 1 /* WC */
9351 || u8Val != 4 /* WT */
9352 || u8Val != 5 /* WP */
9353 || u8Val != 6 /* WB */
9354 || u8Val != 7 /* UC- */)
9355 {
9356 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9357 }
9358 u64Val >>= 3;
9359 }
9360 }
9361
9362 /*
9363 * EFER MSR.
9364 */
9365 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9366 {
9367 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9368 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9369 AssertRCBreak(rc);
9370 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9371 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9372 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9373 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9374 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9375 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9376 VMX_IGS_EFER_LMA_PG_MISMATCH);
9377 }
9378
9379 /*
9380 * Segment registers.
9381 */
9382 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9383 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9384 if (!(u32Eflags & X86_EFL_VM))
9385 {
9386 /* CS */
9387 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9388 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9389 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9390 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9391 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9392 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9393 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9394 /* CS cannot be loaded with NULL in protected mode. */
9395 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9396 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9397 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9398 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9399 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9400 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9401 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9402 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9403 else
9404 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9405
9406 /* SS */
9407 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9408 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9409 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9410 if ( !(pCtx->cr0 & X86_CR0_PE)
9411 || pCtx->cs.Attr.n.u4Type == 3)
9412 {
9413 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9414 }
9415 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9416 {
9417 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9418 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9419 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9420 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9421 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9422 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9423 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9424 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9425 }
9426
9427 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9428 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9429 {
9430 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9431 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9432 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9433 || pCtx->ds.Attr.n.u4Type > 11
9434 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9435 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9436 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9437 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9438 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9439 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9440 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9441 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9442 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9443 }
9444 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9445 {
9446 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9447 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9448 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9449 || pCtx->es.Attr.n.u4Type > 11
9450 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9451 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9452 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9453 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9454 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9455 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9456 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9457 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9458 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9459 }
9460 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9461 {
9462 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9463 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9464 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9465 || pCtx->fs.Attr.n.u4Type > 11
9466 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9467 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9468 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9469 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9470 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9471 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9472 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9473 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9474 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9475 }
9476 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9477 {
9478 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9479 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9480 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9481 || pCtx->gs.Attr.n.u4Type > 11
9482 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9483 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9484 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9485 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9486 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9487 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9488 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9489 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9490 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9491 }
9492 /* 64-bit capable CPUs. */
9493#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9494 if (HMVMX_IS_64BIT_HOST_MODE())
9495 {
9496 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9497 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9498 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9499 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9500 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9501 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9502 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9503 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9504 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9505 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9506 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9507 }
9508#endif
9509 }
9510 else
9511 {
9512 /* V86 mode checks. */
9513 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9514 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9515 {
9516 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9517 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9518 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9519 }
9520 else
9521 {
9522 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9523 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9524 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9525 }
9526
9527 /* CS */
9528 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9529 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9530 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9531 /* SS */
9532 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9533 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9534 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9535 /* DS */
9536 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9537 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9538 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9539 /* ES */
9540 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9541 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9542 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9543 /* FS */
9544 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9545 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9546 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9547 /* GS */
9548 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9549 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9550 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9551 /* 64-bit capable CPUs. */
9552#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9553 if (HMVMX_IS_64BIT_HOST_MODE())
9554 {
9555 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9556 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9557 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9558 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9559 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9560 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9561 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9562 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9563 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9564 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9565 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9566 }
9567#endif
9568 }
9569
9570 /*
9571 * TR.
9572 */
9573 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9574 /* 64-bit capable CPUs. */
9575#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9576 if (HMVMX_IS_64BIT_HOST_MODE())
9577 {
9578 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9579 }
9580#endif
9581 if (fLongModeGuest)
9582 {
9583 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9584 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9585 }
9586 else
9587 {
9588 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9589 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9590 VMX_IGS_TR_ATTR_TYPE_INVALID);
9591 }
9592 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9593 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9594 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9595 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9596 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9597 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9598 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9599 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9600
9601 /*
9602 * GDTR and IDTR.
9603 */
9604#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9605 if (HMVMX_IS_64BIT_HOST_MODE())
9606 {
9607 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9608 AssertRCBreak(rc);
9609 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9610
9611 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9612 AssertRCBreak(rc);
9613 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9614 }
9615#endif
9616
9617 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9618 AssertRCBreak(rc);
9619 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9620
9621 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9622 AssertRCBreak(rc);
9623 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9624
9625 /*
9626 * Guest Non-Register State.
9627 */
9628 /* Activity State. */
9629 uint32_t u32ActivityState;
9630 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9631 AssertRCBreak(rc);
9632 HMVMX_CHECK_BREAK( !u32ActivityState
9633 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9634 VMX_IGS_ACTIVITY_STATE_INVALID);
9635 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9636 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9637 uint32_t u32IntrState;
9638 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9639 AssertRCBreak(rc);
9640 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9641 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9642 {
9643 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9644 }
9645
9646 /** @todo Activity state and injecting interrupts. Left as a todo since we
9647 * currently don't use activity states but ACTIVE. */
9648
9649 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9650 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9651
9652 /* Guest interruptibility-state. */
9653 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9654 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9655 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9656 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9657 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9658 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9659 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9660 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9661 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9662 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9663 {
9664 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9665 {
9666 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9667 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9668 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9669 }
9670 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9671 {
9672 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9673 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9674 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9675 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9676 }
9677 }
9678 /** @todo Assumes the processor is not in SMM. */
9679 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9680 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9681 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9682 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9683 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9684 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9685 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9686 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9687 {
9688 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9689 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9690 }
9691
9692 /* Pending debug exceptions. */
9693 if (HMVMX_IS_64BIT_HOST_MODE())
9694 {
9695 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9696 AssertRCBreak(rc);
9697 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9698 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9699 u32Val = u64Val; /* For pending debug exceptions checks below. */
9700 }
9701 else
9702 {
9703 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9704 AssertRCBreak(rc);
9705 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9706 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9707 }
9708
9709 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9710 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9711 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9712 {
9713 if ( (u32Eflags & X86_EFL_TF)
9714 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9715 {
9716 /* Bit 14 is PendingDebug.BS. */
9717 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9718 }
9719 if ( !(u32Eflags & X86_EFL_TF)
9720 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9721 {
9722 /* Bit 14 is PendingDebug.BS. */
9723 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9724 }
9725 }
9726
9727 /* VMCS link pointer. */
9728 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9729 AssertRCBreak(rc);
9730 if (u64Val != UINT64_C(0xffffffffffffffff))
9731 {
9732 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9733 /** @todo Bits beyond the processor's physical-address width MBZ. */
9734 /** @todo 32-bit located in memory referenced by value of this field (as a
9735 * physical address) must contain the processor's VMCS revision ID. */
9736 /** @todo SMM checks. */
9737 }
9738
9739 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9740 * not using Nested Paging? */
9741 if ( pVM->hm.s.fNestedPaging
9742 && !fLongModeGuest
9743 && CPUMIsGuestInPAEModeEx(pCtx))
9744 {
9745 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9746 AssertRCBreak(rc);
9747 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9748
9749 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9750 AssertRCBreak(rc);
9751 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9752
9753 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9754 AssertRCBreak(rc);
9755 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9756
9757 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9758 AssertRCBreak(rc);
9759 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9760 }
9761
9762 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9763 if (uError == VMX_IGS_ERROR)
9764 uError = VMX_IGS_REASON_NOT_FOUND;
9765 } while (0);
9766
9767 pVCpu->hm.s.u32HMError = uError;
9768 return uError;
9769
9770#undef HMVMX_ERROR_BREAK
9771#undef HMVMX_CHECK_BREAK
9772}
9773
9774/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9775/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9776/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9777
9778/** @name VM-exit handlers.
9779 * @{
9780 */
9781
9782/**
9783 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9784 */
9785HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9786{
9787 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9788 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9789 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9790 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9791 return VINF_SUCCESS;
9792 return VINF_EM_RAW_INTERRUPT;
9793}
9794
9795
9796/**
9797 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9798 */
9799HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9800{
9801 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9802 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9803
9804 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9805 AssertRCReturn(rc, rc);
9806
9807 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9808 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9809 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9810 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9811
9812 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9813 {
9814 /*
9815 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9816 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9817 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9818 *
9819 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9820 */
9821 VMXDispatchHostNmi();
9822 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9823 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9824 return VINF_SUCCESS;
9825 }
9826
9827 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9828 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9829 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9830 {
9831 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9832 return VINF_SUCCESS;
9833 }
9834 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9835 {
9836 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9837 return rc;
9838 }
9839
9840 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9841 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9842 switch (uIntType)
9843 {
9844 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9845 Assert(uVector == X86_XCPT_DB);
9846 /* no break */
9847 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9848 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9849 /* no break */
9850 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9851 {
9852 switch (uVector)
9853 {
9854 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9855 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9856 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9857 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9858 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9859 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9860#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9861 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9862 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9863 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9864 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9865 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9866 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9867 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9868 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9869 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9870 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9871 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9872 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9873#endif
9874 default:
9875 {
9876 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9877 AssertRCReturn(rc, rc);
9878
9879 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9880 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9881 {
9882 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9883 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9884 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9885
9886 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9887 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9888 AssertRCReturn(rc, rc);
9889 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9890 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9891 0 /* GCPtrFaultAddress */);
9892 AssertRCReturn(rc, rc);
9893 }
9894 else
9895 {
9896 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9897 pVCpu->hm.s.u32HMError = uVector;
9898 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9899 }
9900 break;
9901 }
9902 }
9903 break;
9904 }
9905
9906 default:
9907 {
9908 pVCpu->hm.s.u32HMError = uExitIntInfo;
9909 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9910 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9911 break;
9912 }
9913 }
9914 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9915 return rc;
9916}
9917
9918
9919/**
9920 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9921 */
9922HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9923{
9924 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9925
9926 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9927 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9928
9929 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9930 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9931 return VINF_SUCCESS;
9932}
9933
9934
9935/**
9936 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9937 */
9938HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9939{
9940 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9941 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9942 {
9943 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9944 HMVMX_RETURN_UNEXPECTED_EXIT();
9945 }
9946
9947 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9948
9949 /*
9950 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9951 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9952 */
9953 uint32_t uIntrState = 0;
9954 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9955 AssertRCReturn(rc, rc);
9956
9957 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9958 if ( fBlockSti
9959 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9960 {
9961 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9962 }
9963
9964 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9965 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9966
9967 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9968 return VINF_SUCCESS;
9969}
9970
9971
9972/**
9973 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9974 */
9975HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9976{
9977 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9978 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9979 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9980}
9981
9982
9983/**
9984 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9985 */
9986HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9987{
9988 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9990 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9991}
9992
9993
9994/**
9995 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9996 */
9997HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9998{
9999 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10000 PVM pVM = pVCpu->CTX_SUFF(pVM);
10001 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10002 if (RT_LIKELY(rc == VINF_SUCCESS))
10003 {
10004 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10005 Assert(pVmxTransient->cbInstr == 2);
10006 }
10007 else
10008 {
10009 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10010 rc = VERR_EM_INTERPRETER;
10011 }
10012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10013 return rc;
10014}
10015
10016
10017/**
10018 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10019 */
10020HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10021{
10022 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10023 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10024 AssertRCReturn(rc, rc);
10025
10026 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10027 return VINF_EM_RAW_EMULATE_INSTR;
10028
10029 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10030 HMVMX_RETURN_UNEXPECTED_EXIT();
10031}
10032
10033
10034/**
10035 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10036 */
10037HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10038{
10039 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10040 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10041 AssertRCReturn(rc, rc);
10042
10043 PVM pVM = pVCpu->CTX_SUFF(pVM);
10044 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10045 if (RT_LIKELY(rc == VINF_SUCCESS))
10046 {
10047 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10048 Assert(pVmxTransient->cbInstr == 2);
10049 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10050 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10051 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10052 }
10053 else
10054 {
10055 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
10056 rc = VERR_EM_INTERPRETER;
10057 }
10058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10059 return rc;
10060}
10061
10062
10063/**
10064 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10065 */
10066HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10067{
10068 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10069 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10070 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10071 AssertRCReturn(rc, rc);
10072
10073 PVM pVM = pVCpu->CTX_SUFF(pVM);
10074 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10075 if (RT_LIKELY(rc == VINF_SUCCESS))
10076 {
10077 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10078 Assert(pVmxTransient->cbInstr == 3);
10079 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10080 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10081 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10082 }
10083 else
10084 {
10085 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10086 rc = VERR_EM_INTERPRETER;
10087 }
10088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10089 return rc;
10090}
10091
10092
10093/**
10094 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10095 */
10096HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10097{
10098 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10099 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10100 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10101 AssertRCReturn(rc, rc);
10102
10103 PVM pVM = pVCpu->CTX_SUFF(pVM);
10104 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10105 if (RT_LIKELY(rc == VINF_SUCCESS))
10106 {
10107 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10108 Assert(pVmxTransient->cbInstr == 2);
10109 }
10110 else
10111 {
10112 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10113 rc = VERR_EM_INTERPRETER;
10114 }
10115 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10116 return rc;
10117}
10118
10119
10120/**
10121 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10122 */
10123HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10124{
10125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10126
10127 int rc = VERR_NOT_SUPPORTED;
10128 if (GIMAreHypercallsEnabled(pVCpu))
10129 {
10130 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10131 AssertRCReturn(rc, rc);
10132
10133 rc = GIMHypercall(pVCpu, pMixedCtx);
10134 }
10135 if (rc != VINF_SUCCESS)
10136 {
10137 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10138 rc = VINF_SUCCESS;
10139 }
10140
10141 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10142 return rc;
10143}
10144
10145
10146/**
10147 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10148 */
10149HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10150{
10151 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10152 PVM pVM = pVCpu->CTX_SUFF(pVM);
10153 Assert(!pVM->hm.s.fNestedPaging);
10154
10155 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10156 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10157 AssertRCReturn(rc, rc);
10158
10159 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10160 rc = VBOXSTRICTRC_VAL(rc2);
10161 if (RT_LIKELY(rc == VINF_SUCCESS))
10162 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10163 else
10164 {
10165 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10166 pVmxTransient->uExitQualification, rc));
10167 }
10168 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10169 return rc;
10170}
10171
10172
10173/**
10174 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10175 */
10176HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10177{
10178 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10179 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10180 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10181 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10182 AssertRCReturn(rc, rc);
10183
10184 PVM pVM = pVCpu->CTX_SUFF(pVM);
10185 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10186 if (RT_LIKELY(rc == VINF_SUCCESS))
10187 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10188 else
10189 {
10190 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10191 rc = VERR_EM_INTERPRETER;
10192 }
10193 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10194 return rc;
10195}
10196
10197
10198/**
10199 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10200 */
10201HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10202{
10203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10204 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10205 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10206 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10207 AssertRCReturn(rc, rc);
10208
10209 PVM pVM = pVCpu->CTX_SUFF(pVM);
10210 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10211 rc = VBOXSTRICTRC_VAL(rc2);
10212 if (RT_LIKELY( rc == VINF_SUCCESS
10213 || rc == VINF_EM_HALT))
10214 {
10215 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10216 AssertRCReturn(rc3, rc3);
10217
10218 if ( rc == VINF_EM_HALT
10219 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10220 {
10221 rc = VINF_SUCCESS;
10222 }
10223 }
10224 else
10225 {
10226 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10227 rc = VERR_EM_INTERPRETER;
10228 }
10229 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10230 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10231 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10232 return rc;
10233}
10234
10235
10236/**
10237 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10238 */
10239HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10240{
10241 /*
10242 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10243 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10244 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10245 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10246 */
10247 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10248 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10249 HMVMX_RETURN_UNEXPECTED_EXIT();
10250}
10251
10252
10253/**
10254 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10255 */
10256HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10257{
10258 /*
10259 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10260 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10261 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10262 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10263 */
10264 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10265 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10266 HMVMX_RETURN_UNEXPECTED_EXIT();
10267}
10268
10269
10270/**
10271 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10272 */
10273HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10274{
10275 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10276 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10277 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10278 HMVMX_RETURN_UNEXPECTED_EXIT();
10279}
10280
10281
10282/**
10283 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10284 */
10285HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10286{
10287 /*
10288 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10289 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10290 * See Intel spec. 25.3 "Other Causes of VM-exits".
10291 */
10292 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10293 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10294 HMVMX_RETURN_UNEXPECTED_EXIT();
10295}
10296
10297
10298/**
10299 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10300 * VM-exit.
10301 */
10302HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10303{
10304 /*
10305 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10306 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10307 *
10308 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10309 * See Intel spec. "23.8 Restrictions on VMX operation".
10310 */
10311 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10312 return VINF_SUCCESS;
10313}
10314
10315
10316/**
10317 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10318 * VM-exit.
10319 */
10320HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10321{
10322 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10323 return VINF_EM_RESET;
10324}
10325
10326
10327/**
10328 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10329 */
10330HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10331{
10332 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10333 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10334 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10335 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10336 AssertRCReturn(rc, rc);
10337
10338 pMixedCtx->rip++;
10339 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10340 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10341 rc = VINF_SUCCESS;
10342 else
10343 rc = VINF_EM_HALT;
10344
10345 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10346 return rc;
10347}
10348
10349
10350/**
10351 * VM-exit handler for instructions that result in a #UD exception delivered to
10352 * the guest.
10353 */
10354HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10355{
10356 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10357 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10358 return VINF_SUCCESS;
10359}
10360
10361
10362/**
10363 * VM-exit handler for expiry of the VMX preemption timer.
10364 */
10365HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10366{
10367 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10368
10369 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10370 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10371
10372 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10373 PVM pVM = pVCpu->CTX_SUFF(pVM);
10374 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10376 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10377}
10378
10379
10380/**
10381 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10382 */
10383HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10384{
10385 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10386
10387 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10388 /** @todo check if XSETBV is supported by the recompiler. */
10389 return VERR_EM_INTERPRETER;
10390}
10391
10392
10393/**
10394 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10395 */
10396HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10397{
10398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10399
10400 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10401 /** @todo implement EMInterpretInvpcid() */
10402 return VERR_EM_INTERPRETER;
10403}
10404
10405
10406/**
10407 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10408 * Error VM-exit.
10409 */
10410HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10411{
10412 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10413 AssertRCReturn(rc, rc);
10414
10415 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10416 AssertRCReturn(rc, rc);
10417
10418 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10419 NOREF(uInvalidReason);
10420
10421#ifdef VBOX_STRICT
10422 uint32_t uIntrState;
10423 HMVMXHCUINTREG uHCReg;
10424 uint64_t u64Val;
10425 uint32_t u32Val;
10426
10427 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10428 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10429 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10430 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10431 AssertRCReturn(rc, rc);
10432
10433 Log4(("uInvalidReason %u\n", uInvalidReason));
10434 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10435 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10436 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10437 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10438
10439 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10440 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10441 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10442 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10443 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10444 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10445 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10446 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10447 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10448 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10449 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10450 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10451#else
10452 NOREF(pVmxTransient);
10453#endif
10454
10455 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10456 return VERR_VMX_INVALID_GUEST_STATE;
10457}
10458
10459
10460/**
10461 * VM-exit handler for VM-entry failure due to an MSR-load
10462 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10463 */
10464HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10465{
10466 NOREF(pVmxTransient);
10467 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10468 HMVMX_RETURN_UNEXPECTED_EXIT();
10469}
10470
10471
10472/**
10473 * VM-exit handler for VM-entry failure due to a machine-check event
10474 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10475 */
10476HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10477{
10478 NOREF(pVmxTransient);
10479 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10480 HMVMX_RETURN_UNEXPECTED_EXIT();
10481}
10482
10483
10484/**
10485 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10486 * theory.
10487 */
10488HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10489{
10490 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10491 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10492 return VERR_VMX_UNDEFINED_EXIT_CODE;
10493}
10494
10495
10496/**
10497 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10498 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10499 * Conditional VM-exit.
10500 */
10501HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10502{
10503 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10504
10505 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10507 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10508 return VERR_EM_INTERPRETER;
10509 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10510 HMVMX_RETURN_UNEXPECTED_EXIT();
10511}
10512
10513
10514/**
10515 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10516 */
10517HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10518{
10519 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10520
10521 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10522 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10523 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10524 return VERR_EM_INTERPRETER;
10525 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10526 HMVMX_RETURN_UNEXPECTED_EXIT();
10527}
10528
10529
10530/**
10531 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10532 */
10533HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10534{
10535 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10536
10537 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10538 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10539 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10540 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10541 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10542 {
10543 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10544 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10545 }
10546 AssertRCReturn(rc, rc);
10547 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10548
10549#ifdef VBOX_STRICT
10550 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10551 {
10552 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10553 && pMixedCtx->ecx != MSR_K6_EFER)
10554 {
10555 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10556 HMVMX_RETURN_UNEXPECTED_EXIT();
10557 }
10558# if HC_ARCH_BITS == 64
10559 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10560 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10561 {
10562 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10563 HMVMX_RETURN_UNEXPECTED_EXIT();
10564 }
10565# endif
10566 }
10567#endif
10568
10569 PVM pVM = pVCpu->CTX_SUFF(pVM);
10570 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10571 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10572 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10573 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10574 if (RT_LIKELY(rc == VINF_SUCCESS))
10575 {
10576 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10577 Assert(pVmxTransient->cbInstr == 2);
10578 }
10579 return rc;
10580}
10581
10582
10583/**
10584 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10585 */
10586HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10587{
10588 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10589 PVM pVM = pVCpu->CTX_SUFF(pVM);
10590 int rc = VINF_SUCCESS;
10591
10592 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10593 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10594 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10595 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10596 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10597 {
10598 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10599 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10600 }
10601 AssertRCReturn(rc, rc);
10602 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10603
10604 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10605 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10606 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10607
10608 if (RT_LIKELY(rc == VINF_SUCCESS))
10609 {
10610 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10611
10612 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10613 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10614 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10615 {
10616 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10617 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10618 EMInterpretWrmsr() changes it. */
10619 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10620 }
10621 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10622 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10623 else if (pMixedCtx->ecx == MSR_K6_EFER)
10624 {
10625 /*
10626 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10627 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10628 * the other bits as well, SCE and NXE. See @bugref{7368}.
10629 */
10630 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10631 }
10632
10633 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10634 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10635 {
10636 switch (pMixedCtx->ecx)
10637 {
10638 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10639 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10640 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10641 case MSR_K8_FS_BASE: /* no break */
10642 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10643 case MSR_K6_EFER: /* already handled above */ break;
10644 default:
10645 {
10646 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10647 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10648#if HC_ARCH_BITS == 64
10649 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10650 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10651#endif
10652 break;
10653 }
10654 }
10655 }
10656#ifdef VBOX_STRICT
10657 else
10658 {
10659 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10660 switch (pMixedCtx->ecx)
10661 {
10662 case MSR_IA32_SYSENTER_CS:
10663 case MSR_IA32_SYSENTER_EIP:
10664 case MSR_IA32_SYSENTER_ESP:
10665 case MSR_K8_FS_BASE:
10666 case MSR_K8_GS_BASE:
10667 {
10668 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10669 HMVMX_RETURN_UNEXPECTED_EXIT();
10670 }
10671
10672 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10673 default:
10674 {
10675 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10676 {
10677 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10678 if (pMixedCtx->ecx != MSR_K6_EFER)
10679 {
10680 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10681 pMixedCtx->ecx));
10682 HMVMX_RETURN_UNEXPECTED_EXIT();
10683 }
10684 }
10685
10686#if HC_ARCH_BITS == 64
10687 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10688 {
10689 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10690 HMVMX_RETURN_UNEXPECTED_EXIT();
10691 }
10692#endif
10693 break;
10694 }
10695 }
10696 }
10697#endif /* VBOX_STRICT */
10698 }
10699 return rc;
10700}
10701
10702
10703/**
10704 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10705 */
10706HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10707{
10708 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10709
10710 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10711 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10712 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10713 return VERR_EM_INTERPRETER;
10714 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10715 HMVMX_RETURN_UNEXPECTED_EXIT();
10716}
10717
10718
10719/**
10720 * VM-exit handler for when the TPR value is lowered below the specified
10721 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10722 */
10723HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10724{
10725 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10726 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10727
10728 /*
10729 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10730 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10731 * resume guest execution.
10732 */
10733 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10734 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10735 return VINF_SUCCESS;
10736}
10737
10738
10739/**
10740 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10741 * VM-exit.
10742 *
10743 * @retval VINF_SUCCESS when guest execution can continue.
10744 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10745 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10746 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10747 * recompiler.
10748 */
10749HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10750{
10751 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10752 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10753 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10754 AssertRCReturn(rc, rc);
10755
10756 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10757 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10758 PVM pVM = pVCpu->CTX_SUFF(pVM);
10759 switch (uAccessType)
10760 {
10761 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10762 {
10763#if 0
10764 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10765 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10766#else
10767 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10768 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10769 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10770#endif
10771 AssertRCReturn(rc, rc);
10772
10773 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10774 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10775 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10776 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10777
10778 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10779 {
10780 case 0: /* CR0 */
10781 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10782 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10783 break;
10784 case 2: /* CR2 */
10785 /* Nothing to do here, CR2 it's not part of the VMCS. */
10786 break;
10787 case 3: /* CR3 */
10788 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10789 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10790 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10791 break;
10792 case 4: /* CR4 */
10793 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10794 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10795 break;
10796 case 8: /* CR8 */
10797 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10798 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10799 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10800 break;
10801 default:
10802 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10803 break;
10804 }
10805
10806 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10807 break;
10808 }
10809
10810 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10811 {
10812 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10813 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10814 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10815 AssertRCReturn(rc, rc);
10816 Assert( !pVM->hm.s.fNestedPaging
10817 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10818 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10819
10820 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10821 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10822 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10823
10824 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10825 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10826 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10827 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10828 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10829 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10830 break;
10831 }
10832
10833 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10834 {
10835 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10836 AssertRCReturn(rc, rc);
10837 rc = EMInterpretCLTS(pVM, pVCpu);
10838 AssertRCReturn(rc, rc);
10839 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10841 Log4(("CRX CLTS write rc=%d\n", rc));
10842 break;
10843 }
10844
10845 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10846 {
10847 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10848 AssertRCReturn(rc, rc);
10849 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10850 if (RT_LIKELY(rc == VINF_SUCCESS))
10851 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10852 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10853 Log4(("CRX LMSW write rc=%d\n", rc));
10854 break;
10855 }
10856
10857 default:
10858 {
10859 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10860 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10861 }
10862 }
10863
10864 /* Validate possible error codes. */
10865 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10866 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10867 if (RT_SUCCESS(rc))
10868 {
10869 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10870 AssertRCReturn(rc2, rc2);
10871 }
10872
10873 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10874 return rc;
10875}
10876
10877
10878/**
10879 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10880 * VM-exit.
10881 */
10882HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10883{
10884 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10885 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10886
10887 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10888 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10889 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10890 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10891 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10892 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10893 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10894 AssertRCReturn(rc2, rc2);
10895
10896 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10897 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10898 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10899 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10900 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10901 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10902 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10903 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10904
10905 /* I/O operation lookup arrays. */
10906 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10907 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10908
10909 VBOXSTRICTRC rcStrict;
10910 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10911 uint32_t const cbInstr = pVmxTransient->cbInstr;
10912 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10913 PVM pVM = pVCpu->CTX_SUFF(pVM);
10914 if (fIOString)
10915 {
10916#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10917 /*
10918 * INS/OUTS - I/O String instruction.
10919 *
10920 * Use instruction-information if available, otherwise fall back on
10921 * interpreting the instruction.
10922 */
10923 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10924 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10925 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10926 {
10927 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10928 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10929 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10930 AssertRCReturn(rc2, rc2);
10931 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10932 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10933 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10934 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10935 if (fIOWrite)
10936 {
10937 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10938 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10939 }
10940 else
10941 {
10942 /*
10943 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10944 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10945 * See Intel Instruction spec. for "INS".
10946 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10947 */
10948 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10949 }
10950 }
10951 else
10952 {
10953 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10954 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10955 AssertRCReturn(rc2, rc2);
10956 rcStrict = IEMExecOne(pVCpu);
10957 }
10958 /** @todo IEM needs to be setting these flags somehow. */
10959 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10960 fUpdateRipAlready = true;
10961#else
10962 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10963 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10964 if (RT_SUCCESS(rcStrict))
10965 {
10966 if (fIOWrite)
10967 {
10968 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10969 (DISCPUMODE)pDis->uAddrMode, cbValue);
10970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10971 }
10972 else
10973 {
10974 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10975 (DISCPUMODE)pDis->uAddrMode, cbValue);
10976 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10977 }
10978 }
10979 else
10980 {
10981 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10982 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10983 }
10984#endif
10985 }
10986 else
10987 {
10988 /*
10989 * IN/OUT - I/O instruction.
10990 */
10991 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10992 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
10993 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10994 if (fIOWrite)
10995 {
10996 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10997 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10998 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10999 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11000 }
11001 else
11002 {
11003 uint32_t u32Result = 0;
11004 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11005 if (IOM_SUCCESS(rcStrict))
11006 {
11007 /* Save result of I/O IN instr. in AL/AX/EAX. */
11008 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11009 }
11010 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11011 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11013 }
11014 }
11015
11016 if (IOM_SUCCESS(rcStrict))
11017 {
11018 if (!fUpdateRipAlready)
11019 {
11020 pMixedCtx->rip += cbInstr;
11021 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11022 }
11023
11024 /*
11025 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11026 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11027 */
11028 if (fIOString)
11029 {
11030 /** @todo Single-step for INS/OUTS with REP prefix? */
11031 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11032 }
11033 else if (fStepping)
11034 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11035
11036 /*
11037 * If any I/O breakpoints are armed, we need to check if one triggered
11038 * and take appropriate action.
11039 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11040 */
11041 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11042 AssertRCReturn(rc2, rc2);
11043
11044 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11045 * execution engines about whether hyper BPs and such are pending. */
11046 uint32_t const uDr7 = pMixedCtx->dr[7];
11047 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11048 && X86_DR7_ANY_RW_IO(uDr7)
11049 && (pMixedCtx->cr4 & X86_CR4_DE))
11050 || DBGFBpIsHwIoArmed(pVM)))
11051 {
11052 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11053
11054 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11055 VMMRZCallRing3Disable(pVCpu);
11056 HM_DISABLE_PREEMPT_IF_NEEDED();
11057
11058 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
11059
11060 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11061 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11062 {
11063 /* Raise #DB. */
11064 if (fIsGuestDbgActive)
11065 ASMSetDR6(pMixedCtx->dr[6]);
11066 if (pMixedCtx->dr[7] != uDr7)
11067 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11068
11069 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11070 }
11071 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11072 else if ( rcStrict2 != VINF_SUCCESS
11073 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11074 rcStrict = rcStrict2;
11075
11076 HM_RESTORE_PREEMPT_IF_NEEDED();
11077 VMMRZCallRing3Enable(pVCpu);
11078 }
11079 }
11080
11081#ifdef DEBUG
11082 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11083 Assert(!fIOWrite);
11084 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11085 Assert(fIOWrite);
11086 else
11087 {
11088 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11089 * statuses, that the VMM device and some others may return. See
11090 * IOM_SUCCESS() for guidance. */
11091 AssertMsg( RT_FAILURE(rcStrict)
11092 || rcStrict == VINF_SUCCESS
11093 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11094 || rcStrict == VINF_EM_DBG_BREAKPOINT
11095 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11096 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11097 }
11098#endif
11099
11100 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11101 return VBOXSTRICTRC_TODO(rcStrict);
11102}
11103
11104
11105/**
11106 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11107 * VM-exit.
11108 */
11109HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11110{
11111 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11112
11113 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11114 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11115 AssertRCReturn(rc, rc);
11116 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11117 {
11118 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11119 AssertRCReturn(rc, rc);
11120 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11121 {
11122 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11123
11124 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11125 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11126
11127 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11128 Assert(!pVCpu->hm.s.Event.fPending);
11129 pVCpu->hm.s.Event.fPending = true;
11130 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11131 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11132 AssertRCReturn(rc, rc);
11133 if (fErrorCodeValid)
11134 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11135 else
11136 pVCpu->hm.s.Event.u32ErrCode = 0;
11137 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11138 && uVector == X86_XCPT_PF)
11139 {
11140 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11141 }
11142
11143 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11144 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11145 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11146 }
11147 }
11148
11149 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11150 * emulation. */
11151 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11152 return VERR_EM_INTERPRETER;
11153}
11154
11155
11156/**
11157 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11158 */
11159HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11160{
11161 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11162 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11163 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11164 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11165 AssertRCReturn(rc, rc);
11166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11167 return VINF_EM_DBG_STEPPED;
11168}
11169
11170
11171/**
11172 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11173 */
11174HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11175{
11176 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11177
11178 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11179 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11180 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11181 return VINF_SUCCESS;
11182 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11183 return rc;
11184
11185#if 0
11186 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11187 * just sync the whole thing. */
11188 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11189#else
11190 /* Aggressive state sync. for now. */
11191 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11192 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11193 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11194#endif
11195 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11196 AssertRCReturn(rc, rc);
11197
11198 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11199 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11200 switch (uAccessType)
11201 {
11202 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11203 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11204 {
11205 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11206 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11207 {
11208 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11209 }
11210
11211 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11212 GCPhys &= PAGE_BASE_GC_MASK;
11213 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11214 PVM pVM = pVCpu->CTX_SUFF(pVM);
11215 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11216 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11217
11218 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11219 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11220 CPUMCTX2CORE(pMixedCtx), GCPhys);
11221 rc = VBOXSTRICTRC_VAL(rc2);
11222 Log4(("ApicAccess rc=%d\n", rc));
11223 if ( rc == VINF_SUCCESS
11224 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11225 || rc == VERR_PAGE_NOT_PRESENT)
11226 {
11227 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11228 | HM_CHANGED_GUEST_RSP
11229 | HM_CHANGED_GUEST_RFLAGS
11230 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11231 rc = VINF_SUCCESS;
11232 }
11233 break;
11234 }
11235
11236 default:
11237 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11238 rc = VINF_EM_RAW_EMULATE_INSTR;
11239 break;
11240 }
11241
11242 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11243 return rc;
11244}
11245
11246
11247/**
11248 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11249 * VM-exit.
11250 */
11251HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11252{
11253 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11254
11255 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11256 if (pVmxTransient->fWasGuestDebugStateActive)
11257 {
11258 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11259 HMVMX_RETURN_UNEXPECTED_EXIT();
11260 }
11261
11262 int rc = VERR_INTERNAL_ERROR_5;
11263 if ( !DBGFIsStepping(pVCpu)
11264 && !pVCpu->hm.s.fSingleInstruction
11265 && !pVmxTransient->fWasHyperDebugStateActive)
11266 {
11267 /* Don't intercept MOV DRx and #DB any more. */
11268 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11269 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11270 AssertRCReturn(rc, rc);
11271
11272 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11273 {
11274#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11275 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11276 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11277 AssertRCReturn(rc, rc);
11278#endif
11279 }
11280
11281 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11282 VMMRZCallRing3Disable(pVCpu);
11283 HM_DISABLE_PREEMPT_IF_NEEDED();
11284
11285 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11286 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11287 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11288
11289 HM_RESTORE_PREEMPT_IF_NEEDED();
11290 VMMRZCallRing3Enable(pVCpu);
11291
11292#ifdef VBOX_WITH_STATISTICS
11293 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11294 AssertRCReturn(rc, rc);
11295 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11297 else
11298 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11299#endif
11300 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11301 return VINF_SUCCESS;
11302 }
11303
11304 /*
11305 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11306 * Update the segment registers and DR7 from the CPU.
11307 */
11308 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11309 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11310 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11311 AssertRCReturn(rc, rc);
11312 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11313
11314 PVM pVM = pVCpu->CTX_SUFF(pVM);
11315 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11316 {
11317 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11318 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11319 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11320 if (RT_SUCCESS(rc))
11321 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11322 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11323 }
11324 else
11325 {
11326 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11327 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11328 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11330 }
11331
11332 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11333 if (RT_SUCCESS(rc))
11334 {
11335 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11336 AssertRCReturn(rc2, rc2);
11337 }
11338 return rc;
11339}
11340
11341
11342/**
11343 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11344 * Conditional VM-exit.
11345 */
11346HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11347{
11348 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11349 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11350
11351 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11352 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11353 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11354 return VINF_SUCCESS;
11355 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11356 return rc;
11357
11358 RTGCPHYS GCPhys = 0;
11359 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11360
11361#if 0
11362 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11363#else
11364 /* Aggressive state sync. for now. */
11365 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11366 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11367 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11368#endif
11369 AssertRCReturn(rc, rc);
11370
11371 /*
11372 * If we succeed, resume guest execution.
11373 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11374 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11375 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11376 * weird case. See @bugref{6043}.
11377 */
11378 PVM pVM = pVCpu->CTX_SUFF(pVM);
11379 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11380 rc = VBOXSTRICTRC_VAL(rc2);
11381 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11382 if ( rc == VINF_SUCCESS
11383 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11384 || rc == VERR_PAGE_NOT_PRESENT)
11385 {
11386 /* Successfully handled MMIO operation. */
11387 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11388 | HM_CHANGED_GUEST_RSP
11389 | HM_CHANGED_GUEST_RFLAGS
11390 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11391 rc = VINF_SUCCESS;
11392 }
11393 return rc;
11394}
11395
11396
11397/**
11398 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11399 * VM-exit.
11400 */
11401HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11402{
11403 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11404 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11405
11406 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11407 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11408 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11409 return VINF_SUCCESS;
11410 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11411 return rc;
11412
11413 RTGCPHYS GCPhys = 0;
11414 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11415 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11416#if 0
11417 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11418#else
11419 /* Aggressive state sync. for now. */
11420 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11421 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11422 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11423#endif
11424 AssertRCReturn(rc, rc);
11425
11426 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11427 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11428
11429 RTGCUINT uErrorCode = 0;
11430 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11431 uErrorCode |= X86_TRAP_PF_ID;
11432 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11433 uErrorCode |= X86_TRAP_PF_RW;
11434 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11435 uErrorCode |= X86_TRAP_PF_P;
11436
11437 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11438
11439 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11440 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11441
11442 /* Handle the pagefault trap for the nested shadow table. */
11443 PVM pVM = pVCpu->CTX_SUFF(pVM);
11444 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11445 TRPMResetTrap(pVCpu);
11446
11447 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11448 if ( rc == VINF_SUCCESS
11449 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11450 || rc == VERR_PAGE_NOT_PRESENT)
11451 {
11452 /* Successfully synced our nested page tables. */
11453 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11454 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11455 | HM_CHANGED_GUEST_RSP
11456 | HM_CHANGED_GUEST_RFLAGS);
11457 return VINF_SUCCESS;
11458 }
11459
11460 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11461 return rc;
11462}
11463
11464/** @} */
11465
11466/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11467/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11468/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11469
11470/** @name VM-exit exception handlers.
11471 * @{
11472 */
11473
11474/**
11475 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11476 */
11477static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11478{
11479 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11481
11482 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11483 AssertRCReturn(rc, rc);
11484
11485 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11486 {
11487 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11488 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11489
11490 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11491 * provides VM-exit instruction length. If this causes problem later,
11492 * disassemble the instruction like it's done on AMD-V. */
11493 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11494 AssertRCReturn(rc2, rc2);
11495 return rc;
11496 }
11497
11498 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11499 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11500 return rc;
11501}
11502
11503
11504/**
11505 * VM-exit exception handler for #BP (Breakpoint exception).
11506 */
11507static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11508{
11509 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11510 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11511
11512 /** @todo Try optimize this by not saving the entire guest state unless
11513 * really needed. */
11514 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11515 AssertRCReturn(rc, rc);
11516
11517 PVM pVM = pVCpu->CTX_SUFF(pVM);
11518 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11519 if (rc == VINF_EM_RAW_GUEST_TRAP)
11520 {
11521 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11522 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11523 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11524 AssertRCReturn(rc, rc);
11525
11526 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11527 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11528 }
11529
11530 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11531 return rc;
11532}
11533
11534
11535/**
11536 * VM-exit exception handler for #DB (Debug exception).
11537 */
11538static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11539{
11540 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11541 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11542 Log6(("XcptDB\n"));
11543
11544 /*
11545 * Get the DR6-like values from the exit qualification and pass it to DBGF
11546 * for processing.
11547 */
11548 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11549 AssertRCReturn(rc, rc);
11550
11551 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11552 uint64_t uDR6 = X86_DR6_INIT_VAL;
11553 uDR6 |= ( pVmxTransient->uExitQualification
11554 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11555
11556 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11557 if (rc == VINF_EM_RAW_GUEST_TRAP)
11558 {
11559 /*
11560 * The exception was for the guest. Update DR6, DR7.GD and
11561 * IA32_DEBUGCTL.LBR before forwarding it.
11562 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11563 */
11564 VMMRZCallRing3Disable(pVCpu);
11565 HM_DISABLE_PREEMPT_IF_NEEDED();
11566
11567 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11568 pMixedCtx->dr[6] |= uDR6;
11569 if (CPUMIsGuestDebugStateActive(pVCpu))
11570 ASMSetDR6(pMixedCtx->dr[6]);
11571
11572 HM_RESTORE_PREEMPT_IF_NEEDED();
11573 VMMRZCallRing3Enable(pVCpu);
11574
11575 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11576 AssertRCReturn(rc, rc);
11577
11578 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11579 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11580
11581 /* Paranoia. */
11582 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11583 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11584
11585 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11586 AssertRCReturn(rc, rc);
11587
11588 /*
11589 * Raise #DB in the guest.
11590 *
11591 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11592 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11593 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11594 *
11595 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11596 */
11597 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11598 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11599 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11600 AssertRCReturn(rc, rc);
11601 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11602 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11603 return VINF_SUCCESS;
11604 }
11605
11606 /*
11607 * Not a guest trap, must be a hypervisor related debug event then.
11608 * Update DR6 in case someone is interested in it.
11609 */
11610 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11611 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11612 CPUMSetHyperDR6(pVCpu, uDR6);
11613
11614 return rc;
11615}
11616
11617
11618/**
11619 * VM-exit exception handler for #NM (Device-not-available exception: floating
11620 * point exception).
11621 */
11622static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11623{
11624 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11625
11626 /* We require CR0 and EFER. EFER is always up-to-date. */
11627 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11628 AssertRCReturn(rc, rc);
11629
11630 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11631 VMMRZCallRing3Disable(pVCpu);
11632 HM_DISABLE_PREEMPT_IF_NEEDED();
11633
11634 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11635 if (pVmxTransient->fWasGuestFPUStateActive)
11636 {
11637 rc = VINF_EM_RAW_GUEST_TRAP;
11638 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11639 }
11640 else
11641 {
11642#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11643 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11644#endif
11645 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11646 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11647 }
11648
11649 HM_RESTORE_PREEMPT_IF_NEEDED();
11650 VMMRZCallRing3Enable(pVCpu);
11651
11652 if (rc == VINF_SUCCESS)
11653 {
11654 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11655 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11656 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11657 pVCpu->hm.s.fUseGuestFpu = true;
11658 }
11659 else
11660 {
11661 /* Forward #NM to the guest. */
11662 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11663 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11664 AssertRCReturn(rc, rc);
11665 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11666 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11667 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11668 }
11669
11670 return VINF_SUCCESS;
11671}
11672
11673
11674/**
11675 * VM-exit exception handler for #GP (General-protection exception).
11676 *
11677 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11678 */
11679static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11680{
11681 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11683
11684 int rc = VERR_INTERNAL_ERROR_5;
11685 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11686 {
11687#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11688 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11689 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11690 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11691 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11692 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11693 AssertRCReturn(rc, rc);
11694 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11695 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11696 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11697 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11698 return rc;
11699#else
11700 /* We don't intercept #GP. */
11701 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11702 NOREF(pVmxTransient);
11703 return VERR_VMX_UNEXPECTED_EXCEPTION;
11704#endif
11705 }
11706
11707 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11708 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11709
11710 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11711 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11712 AssertRCReturn(rc, rc);
11713
11714 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11715 uint32_t cbOp = 0;
11716 PVM pVM = pVCpu->CTX_SUFF(pVM);
11717 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11718 if (RT_SUCCESS(rc))
11719 {
11720 rc = VINF_SUCCESS;
11721 Assert(cbOp == pDis->cbInstr);
11722 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11723 switch (pDis->pCurInstr->uOpcode)
11724 {
11725 case OP_CLI:
11726 {
11727 pMixedCtx->eflags.Bits.u1IF = 0;
11728 pMixedCtx->rip += pDis->cbInstr;
11729 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11730 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11731 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11732 break;
11733 }
11734
11735 case OP_STI:
11736 {
11737 pMixedCtx->eflags.Bits.u1IF = 1;
11738 pMixedCtx->rip += pDis->cbInstr;
11739 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11740 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11741 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11742 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11743 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11744 break;
11745 }
11746
11747 case OP_HLT:
11748 {
11749 rc = VINF_EM_HALT;
11750 pMixedCtx->rip += pDis->cbInstr;
11751 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11753 break;
11754 }
11755
11756 case OP_POPF:
11757 {
11758 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11759 uint32_t cbParm;
11760 uint32_t uMask;
11761 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11762 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11763 {
11764 cbParm = 4;
11765 uMask = 0xffffffff;
11766 }
11767 else
11768 {
11769 cbParm = 2;
11770 uMask = 0xffff;
11771 }
11772
11773 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11774 RTGCPTR GCPtrStack = 0;
11775 X86EFLAGS Eflags;
11776 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11777 &GCPtrStack);
11778 if (RT_SUCCESS(rc))
11779 {
11780 Assert(sizeof(Eflags.u32) >= cbParm);
11781 Eflags.u32 = 0;
11782 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11783 }
11784 if (RT_FAILURE(rc))
11785 {
11786 rc = VERR_EM_INTERPRETER;
11787 break;
11788 }
11789 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11790 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11791 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11792 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11793 pMixedCtx->esp += cbParm;
11794 pMixedCtx->esp &= uMask;
11795 pMixedCtx->rip += pDis->cbInstr;
11796 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11797 | HM_CHANGED_GUEST_RSP
11798 | HM_CHANGED_GUEST_RFLAGS);
11799 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11800 if (fStepping)
11801 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11802
11803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11804 break;
11805 }
11806
11807 case OP_PUSHF:
11808 {
11809 uint32_t cbParm;
11810 uint32_t uMask;
11811 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11812 {
11813 cbParm = 4;
11814 uMask = 0xffffffff;
11815 }
11816 else
11817 {
11818 cbParm = 2;
11819 uMask = 0xffff;
11820 }
11821
11822 /* Get the stack pointer & push the contents of eflags onto the stack. */
11823 RTGCPTR GCPtrStack = 0;
11824 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11825 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11826 if (RT_FAILURE(rc))
11827 {
11828 rc = VERR_EM_INTERPRETER;
11829 break;
11830 }
11831 X86EFLAGS Eflags = pMixedCtx->eflags;
11832 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11833 Eflags.Bits.u1RF = 0;
11834 Eflags.Bits.u1VM = 0;
11835
11836 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11837 if (RT_FAILURE(rc))
11838 {
11839 rc = VERR_EM_INTERPRETER;
11840 break;
11841 }
11842 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11843 pMixedCtx->esp -= cbParm;
11844 pMixedCtx->esp &= uMask;
11845 pMixedCtx->rip += pDis->cbInstr;
11846 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11847 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11848 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11849 break;
11850 }
11851
11852 case OP_IRET:
11853 {
11854 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11855 * instruction reference. */
11856 RTGCPTR GCPtrStack = 0;
11857 uint32_t uMask = 0xffff;
11858 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11859 uint16_t aIretFrame[3];
11860 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11861 {
11862 rc = VERR_EM_INTERPRETER;
11863 break;
11864 }
11865 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11866 &GCPtrStack);
11867 if (RT_SUCCESS(rc))
11868 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11869 if (RT_FAILURE(rc))
11870 {
11871 rc = VERR_EM_INTERPRETER;
11872 break;
11873 }
11874 pMixedCtx->eip = 0;
11875 pMixedCtx->ip = aIretFrame[0];
11876 pMixedCtx->cs.Sel = aIretFrame[1];
11877 pMixedCtx->cs.ValidSel = aIretFrame[1];
11878 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11879 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11880 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11881 pMixedCtx->sp += sizeof(aIretFrame);
11882 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11883 | HM_CHANGED_GUEST_SEGMENT_REGS
11884 | HM_CHANGED_GUEST_RSP
11885 | HM_CHANGED_GUEST_RFLAGS);
11886 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11887 if (fStepping)
11888 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11889 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11890 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11891 break;
11892 }
11893
11894 case OP_INT:
11895 {
11896 uint16_t uVector = pDis->Param1.uValue & 0xff;
11897 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11898 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11899 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11900 break;
11901 }
11902
11903 case OP_INTO:
11904 {
11905 if (pMixedCtx->eflags.Bits.u1OF)
11906 {
11907 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11908 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11909 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11910 }
11911 break;
11912 }
11913
11914 default:
11915 {
11916 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11917 EMCODETYPE_SUPERVISOR);
11918 rc = VBOXSTRICTRC_VAL(rc2);
11919 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11920 /** @todo We have to set pending-debug exceptions here when the guest is
11921 * single-stepping depending on the instruction that was interpreted. */
11922 Log4(("#GP rc=%Rrc\n", rc));
11923 break;
11924 }
11925 }
11926 }
11927 else
11928 rc = VERR_EM_INTERPRETER;
11929
11930 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11931 ("#GP Unexpected rc=%Rrc\n", rc));
11932 return rc;
11933}
11934
11935
11936#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11937/**
11938 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11939 * the exception reported in the VMX transient structure back into the VM.
11940 *
11941 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11942 * up-to-date.
11943 */
11944static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11945{
11946 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11947
11948 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11949 hmR0VmxCheckExitDueToEventDelivery(). */
11950 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11951 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11952 AssertRCReturn(rc, rc);
11953 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11954
11955#ifdef DEBUG_ramshankar
11956 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11957 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11958 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11959#endif
11960
11961 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11962 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11963 return VINF_SUCCESS;
11964}
11965#endif
11966
11967
11968/**
11969 * VM-exit exception handler for #PF (Page-fault exception).
11970 */
11971static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11972{
11973 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11974 PVM pVM = pVCpu->CTX_SUFF(pVM);
11975 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11976 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11977 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11978 AssertRCReturn(rc, rc);
11979
11980#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11981 if (pVM->hm.s.fNestedPaging)
11982 {
11983 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11984 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11985 {
11986 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11987 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11988 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11989 }
11990 else
11991 {
11992 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11993 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11994 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11995 }
11996 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11997 return rc;
11998 }
11999#else
12000 Assert(!pVM->hm.s.fNestedPaging);
12001 NOREF(pVM);
12002#endif
12003
12004 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12005 AssertRCReturn(rc, rc);
12006
12007 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12008 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12009
12010 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12011 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12012 (RTGCPTR)pVmxTransient->uExitQualification);
12013
12014 Log4(("#PF: rc=%Rrc\n", rc));
12015 if (rc == VINF_SUCCESS)
12016 {
12017 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12018 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12019 * memory? We don't update the whole state here... */
12020 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12021 | HM_CHANGED_GUEST_RSP
12022 | HM_CHANGED_GUEST_RFLAGS
12023 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12024 TRPMResetTrap(pVCpu);
12025 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12026 return rc;
12027 }
12028 else if (rc == VINF_EM_RAW_GUEST_TRAP)
12029 {
12030 if (!pVmxTransient->fVectoringPF)
12031 {
12032 /* It's a guest page fault and needs to be reflected to the guest. */
12033 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12034 TRPMResetTrap(pVCpu);
12035 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12036 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12037 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12038 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12039 }
12040 else
12041 {
12042 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12043 TRPMResetTrap(pVCpu);
12044 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12045 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12046 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12047 }
12048
12049 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12050 return VINF_SUCCESS;
12051 }
12052
12053 TRPMResetTrap(pVCpu);
12054 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12055 return rc;
12056}
12057
12058/** @} */
12059
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette