VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 52147

Last change on this file since 52147 was 52147, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: Don't lose interruptibility-state when we get preempted early!
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 500.1 KB
Line 
1/* $Id: HMVMXR0.cpp 52147 2014-07-23 09:57:19Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns VBox status code.
323 * @param pVCpu Pointer to the VMCPU.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337
338/*******************************************************************************
339* Internal Functions *
340*******************************************************************************/
341static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
342static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
343static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
344 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
345#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
346static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
347#endif
348#ifndef HMVMX_USE_FUNCTION_TABLE
349DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
350# define HMVMX_EXIT_DECL static int
351#else
352# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
353#endif
354
355/** @name VM-exit handlers.
356 * @{
357 */
358static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
359static FNVMXEXITHANDLER hmR0VmxExitExtInt;
360static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
361static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
362static FNVMXEXITHANDLER hmR0VmxExitSipi;
363static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
364static FNVMXEXITHANDLER hmR0VmxExitSmi;
365static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
366static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
367static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
368static FNVMXEXITHANDLER hmR0VmxExitCpuid;
369static FNVMXEXITHANDLER hmR0VmxExitGetsec;
370static FNVMXEXITHANDLER hmR0VmxExitHlt;
371static FNVMXEXITHANDLER hmR0VmxExitInvd;
372static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
373static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
374static FNVMXEXITHANDLER hmR0VmxExitVmcall;
375static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
376static FNVMXEXITHANDLER hmR0VmxExitRsm;
377static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
378static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
379static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
380static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
381static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
382static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
383static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
384static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
385static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
386static FNVMXEXITHANDLER hmR0VmxExitMwait;
387static FNVMXEXITHANDLER hmR0VmxExitMtf;
388static FNVMXEXITHANDLER hmR0VmxExitMonitor;
389static FNVMXEXITHANDLER hmR0VmxExitPause;
390static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
391static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
392static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
395static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
396static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
397static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
398static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
399static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
400static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
401static FNVMXEXITHANDLER hmR0VmxExitRdrand;
402static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
403/** @} */
404
405static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
412static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413#endif
414static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
415
416/*******************************************************************************
417* Global Variables *
418*******************************************************************************/
419#ifdef HMVMX_USE_FUNCTION_TABLE
420
421/**
422 * VMX_EXIT dispatch table.
423 */
424static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
425{
426 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
427 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
428 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
429 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
430 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
431 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
432 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
433 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
434 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
435 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
436 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
437 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
438 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
439 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
440 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
441 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
442 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
443 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
444 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
445 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
446 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
447 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
448 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
449 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
450 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
451 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
452 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
453 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
454 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
455 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
456 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
457 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
458 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
459 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
460 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
461 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
462 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
463 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
464 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
465 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
466 /* 40 UNDEFINED */ hmR0VmxExitPause,
467 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
468 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
469 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
470 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
471 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
473 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
475 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
476 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
477 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
478 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
479 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
480 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
481 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
482 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
483 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
484 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
485 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
486};
487#endif /* HMVMX_USE_FUNCTION_TABLE */
488
489#ifdef VBOX_STRICT
490static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
491{
492 /* 0 */ "(Not Used)",
493 /* 1 */ "VMCALL executed in VMX root operation.",
494 /* 2 */ "VMCLEAR with invalid physical address.",
495 /* 3 */ "VMCLEAR with VMXON pointer.",
496 /* 4 */ "VMLAUNCH with non-clear VMCS.",
497 /* 5 */ "VMRESUME with non-launched VMCS.",
498 /* 6 */ "VMRESUME after VMXOFF",
499 /* 7 */ "VM entry with invalid control fields.",
500 /* 8 */ "VM entry with invalid host state fields.",
501 /* 9 */ "VMPTRLD with invalid physical address.",
502 /* 10 */ "VMPTRLD with VMXON pointer.",
503 /* 11 */ "VMPTRLD with incorrect revision identifier.",
504 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
505 /* 13 */ "VMWRITE to read-only VMCS component.",
506 /* 14 */ "(Not Used)",
507 /* 15 */ "VMXON executed in VMX root operation.",
508 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
509 /* 17 */ "VM entry with non-launched executing VMCS.",
510 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
511 /* 19 */ "VMCALL with non-clear VMCS.",
512 /* 20 */ "VMCALL with invalid VM-exit control fields.",
513 /* 21 */ "(Not Used)",
514 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
515 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
516 /* 24 */ "VMCALL with invalid SMM-monitor features.",
517 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
518 /* 26 */ "VM entry with events blocked by MOV SS.",
519 /* 27 */ "(Not Used)",
520 /* 28 */ "Invalid operand to INVEPT/INVVPID."
521};
522#endif /* VBOX_STRICT */
523
524
525
526/**
527 * Updates the VM's last error record. If there was a VMX instruction error,
528 * reads the error data from the VMCS and updates VCPU's last error record as
529 * well.
530 *
531 * @param pVM Pointer to the VM.
532 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
533 * VERR_VMX_UNABLE_TO_START_VM or
534 * VERR_VMX_INVALID_VMCS_FIELD).
535 * @param rc The error code.
536 */
537static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
538{
539 AssertPtr(pVM);
540 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
541 || rc == VERR_VMX_UNABLE_TO_START_VM)
542 {
543 AssertPtrReturnVoid(pVCpu);
544 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
545 }
546 pVM->hm.s.lLastError = rc;
547}
548
549
550/**
551 * Reads the VM-entry interruption-information field from the VMCS into the VMX
552 * transient structure.
553 *
554 * @returns VBox status code.
555 * @param pVmxTransient Pointer to the VMX transient structure.
556 *
557 * @remarks No-long-jump zone!!!
558 */
559DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
560{
561 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
562 AssertRCReturn(rc, rc);
563 return VINF_SUCCESS;
564}
565
566
567/**
568 * Reads the VM-entry exception error code field from the VMCS into
569 * the VMX transient structure.
570 *
571 * @returns VBox status code.
572 * @param pVmxTransient Pointer to the VMX transient structure.
573 *
574 * @remarks No-long-jump zone!!!
575 */
576DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
577{
578 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
579 AssertRCReturn(rc, rc);
580 return VINF_SUCCESS;
581}
582
583
584/**
585 * Reads the VM-entry exception error code field from the VMCS into
586 * the VMX transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600
601/**
602 * Reads the VM-exit interruption-information field from the VMCS into the VMX
603 * transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 */
608DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
611 {
612 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
613 AssertRCReturn(rc, rc);
614 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
615 }
616 return VINF_SUCCESS;
617}
618
619
620/**
621 * Reads the VM-exit interruption error code from the VMCS into the VMX
622 * transient structure.
623 *
624 * @returns VBox status code.
625 * @param pVmxTransient Pointer to the VMX transient structure.
626 */
627DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
630 {
631 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
632 AssertRCReturn(rc, rc);
633 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
634 }
635 return VINF_SUCCESS;
636}
637
638
639/**
640 * Reads the VM-exit instruction length field from the VMCS into the VMX
641 * transient structure.
642 *
643 * @returns VBox status code.
644 * @param pVCpu Pointer to the VMCPU.
645 * @param pVmxTransient Pointer to the VMX transient structure.
646 */
647DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
648{
649 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
650 {
651 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
652 AssertRCReturn(rc, rc);
653 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
654 }
655 return VINF_SUCCESS;
656}
657
658
659/**
660 * Reads the VM-exit instruction-information field from the VMCS into
661 * the VMX transient structure.
662 *
663 * @returns VBox status code.
664 * @param pVmxTransient Pointer to the VMX transient structure.
665 */
666DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
667{
668 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
669 {
670 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
671 AssertRCReturn(rc, rc);
672 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
673 }
674 return VINF_SUCCESS;
675}
676
677
678/**
679 * Reads the exit qualification from the VMCS into the VMX transient structure.
680 *
681 * @returns VBox status code.
682 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
683 * case).
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
689 {
690 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the IDT-vectoring information field from the VMCS into the VMX
700 * transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 *
705 * @remarks No-long-jump zone!!!
706 */
707DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the IDT-vectoring error code from the VMCS into the VMX
721 * transient structure.
722 *
723 * @returns VBox status code.
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
729 {
730 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Enters VMX root mode operation on the current CPU.
740 *
741 * @returns VBox status code.
742 * @param pVM Pointer to the VM (optional, can be NULL, after
743 * a resume).
744 * @param HCPhysCpuPage Physical address of the VMXON region.
745 * @param pvCpuPage Pointer to the VMXON region.
746 */
747static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
748{
749 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
750 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
751 Assert(pvCpuPage);
752 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
753
754 if (pVM)
755 {
756 /* Write the VMCS revision dword to the VMXON region. */
757 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
758 }
759
760 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
761 RTCCUINTREG uEflags = ASMIntDisableFlags();
762
763 /* Enable the VMX bit in CR4 if necessary. */
764 RTCCUINTREG uCr4 = ASMGetCR4();
765 if (!(uCr4 & X86_CR4_VMXE))
766 ASMSetCR4(uCr4 | X86_CR4_VMXE);
767
768 /* Enter VMX root mode. */
769 int rc = VMXEnable(HCPhysCpuPage);
770 if (RT_FAILURE(rc))
771 ASMSetCR4(uCr4);
772
773 /* Restore interrupts. */
774 ASMSetFlags(uEflags);
775 return rc;
776}
777
778
779/**
780 * Exits VMX root mode operation on the current CPU.
781 *
782 * @returns VBox status code.
783 */
784static int hmR0VmxLeaveRootMode(void)
785{
786 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
787
788 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
789 RTCCUINTREG uEflags = ASMIntDisableFlags();
790
791 /* If we're for some reason not in VMX root mode, then don't leave it. */
792 RTCCUINTREG uHostCR4 = ASMGetCR4();
793
794 int rc;
795 if (uHostCR4 & X86_CR4_VMXE)
796 {
797 /* Exit VMX root mode and clear the VMX bit in CR4. */
798 VMXDisable();
799 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
800 rc = VINF_SUCCESS;
801 }
802 else
803 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
804
805 /* Restore interrupts. */
806 ASMSetFlags(uEflags);
807 return rc;
808}
809
810
811/**
812 * Allocates and maps one physically contiguous page. The allocated page is
813 * zero'd out. (Used by various VT-x structures).
814 *
815 * @returns IPRT status code.
816 * @param pMemObj Pointer to the ring-0 memory object.
817 * @param ppVirt Where to store the virtual address of the
818 * allocation.
819 * @param pPhys Where to store the physical address of the
820 * allocation.
821 */
822DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
823{
824 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
825 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
826 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
827
828 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
829 if (RT_FAILURE(rc))
830 return rc;
831 *ppVirt = RTR0MemObjAddress(*pMemObj);
832 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
833 ASMMemZero32(*ppVirt, PAGE_SIZE);
834 return VINF_SUCCESS;
835}
836
837
838/**
839 * Frees and unmaps an allocated physical page.
840 *
841 * @param pMemObj Pointer to the ring-0 memory object.
842 * @param ppVirt Where to re-initialize the virtual address of
843 * allocation as 0.
844 * @param pHCPhys Where to re-initialize the physical address of the
845 * allocation as 0.
846 */
847DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
848{
849 AssertPtr(pMemObj);
850 AssertPtr(ppVirt);
851 AssertPtr(pHCPhys);
852 if (*pMemObj != NIL_RTR0MEMOBJ)
853 {
854 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
855 AssertRC(rc);
856 *pMemObj = NIL_RTR0MEMOBJ;
857 *ppVirt = 0;
858 *pHCPhys = 0;
859 }
860}
861
862
863/**
864 * Worker function to free VT-x related structures.
865 *
866 * @returns IPRT status code.
867 * @param pVM Pointer to the VM.
868 */
869static void hmR0VmxStructsFree(PVM pVM)
870{
871 for (VMCPUID i = 0; i < pVM->cCpus; i++)
872 {
873 PVMCPU pVCpu = &pVM->aCpus[i];
874 AssertPtr(pVCpu);
875
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
878
879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
881
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
884 }
885
886 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
887#ifdef VBOX_WITH_CRASHDUMP_MAGIC
888 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
889#endif
890}
891
892
893/**
894 * Worker function to allocate VT-x related VM structures.
895 *
896 * @returns IPRT status code.
897 * @param pVM Pointer to the VM.
898 */
899static int hmR0VmxStructsAlloc(PVM pVM)
900{
901 /*
902 * Initialize members up-front so we can cleanup properly on allocation failure.
903 */
904#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVM->hm.s.vmx.HCPhys##a_Name = 0;
908
909#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
910 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
911 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
912 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
913
914#ifdef VBOX_WITH_CRASHDUMP_MAGIC
915 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
916#endif
917 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
918
919 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
925 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
926 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
927 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
928 }
929#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
930#undef VMXLOCAL_INIT_VM_MEMOBJ
931
932 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
933 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
934 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
935 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
936
937 /*
938 * Allocate all the VT-x structures.
939 */
940 int rc = VINF_SUCCESS;
941#ifdef VBOX_WITH_CRASHDUMP_MAGIC
942 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
943 if (RT_FAILURE(rc))
944 goto cleanup;
945 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
946 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
947#endif
948
949 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
950 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
951 {
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
953 &pVM->hm.s.vmx.HCPhysApicAccess);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 }
957
958 /*
959 * Initialize per-VCPU VT-x structures.
960 */
961 for (VMCPUID i = 0; i < pVM->cCpus; i++)
962 {
963 PVMCPU pVCpu = &pVM->aCpus[i];
964 AssertPtr(pVCpu);
965
966 /* Allocate the VM control structure (VMCS). */
967 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
968 if (RT_FAILURE(rc))
969 goto cleanup;
970
971 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
972 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
973 {
974 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
975 &pVCpu->hm.s.vmx.HCPhysVirtApic);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978 }
979
980 /*
981 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
982 * transparent accesses of specific MSRs.
983 *
984 * If the condition for enabling MSR bitmaps changes here, don't forget to
985 * update HMIsMsrBitmapsAvailable().
986 */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
988 {
989 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
990 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
994 }
995
996 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
997 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000
1001 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1002 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1003 if (RT_FAILURE(rc))
1004 goto cleanup;
1005 }
1006
1007 return VINF_SUCCESS;
1008
1009cleanup:
1010 hmR0VmxStructsFree(pVM);
1011 return rc;
1012}
1013
1014
1015/**
1016 * Does global VT-x initialization (called during module initialization).
1017 *
1018 * @returns VBox status code.
1019 */
1020VMMR0DECL(int) VMXR0GlobalInit(void)
1021{
1022#ifdef HMVMX_USE_FUNCTION_TABLE
1023 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1024# ifdef VBOX_STRICT
1025 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1026 Assert(g_apfnVMExitHandlers[i]);
1027# endif
1028#endif
1029 return VINF_SUCCESS;
1030}
1031
1032
1033/**
1034 * Does global VT-x termination (called during module termination).
1035 */
1036VMMR0DECL(void) VMXR0GlobalTerm()
1037{
1038 /* Nothing to do currently. */
1039}
1040
1041
1042/**
1043 * Sets up and activates VT-x on the current CPU.
1044 *
1045 * @returns VBox status code.
1046 * @param pCpu Pointer to the global CPU info struct.
1047 * @param pVM Pointer to the VM (can be NULL after a host resume
1048 * operation).
1049 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1050 * fEnabledByHost is true).
1051 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1052 * @a fEnabledByHost is true).
1053 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1054 * enable VT-x on the host.
1055 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1056 */
1057VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1058 void *pvMsrs)
1059{
1060 Assert(pCpu);
1061 Assert(pvMsrs);
1062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1063
1064 /* Enable VT-x if it's not already enabled by the host. */
1065 if (!fEnabledByHost)
1066 {
1067 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1068 if (RT_FAILURE(rc))
1069 return rc;
1070 }
1071
1072 /*
1073 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1074 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1075 */
1076 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1077 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1078 {
1079 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1080 pCpu->fFlushAsidBeforeUse = false;
1081 }
1082 else
1083 pCpu->fFlushAsidBeforeUse = true;
1084
1085 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1086 ++pCpu->cTlbFlushes;
1087
1088 return VINF_SUCCESS;
1089}
1090
1091
1092/**
1093 * Deactivates VT-x on the current CPU.
1094 *
1095 * @returns VBox status code.
1096 * @param pCpu Pointer to the global CPU info struct.
1097 * @param pvCpuPage Pointer to the VMXON region.
1098 * @param HCPhysCpuPage Physical address of the VMXON region.
1099 *
1100 * @remarks This function should never be called when SUPR0EnableVTx() or
1101 * similar was used to enable VT-x on the host.
1102 */
1103VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1104{
1105 NOREF(pCpu);
1106 NOREF(pvCpuPage);
1107 NOREF(HCPhysCpuPage);
1108
1109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1110 return hmR0VmxLeaveRootMode();
1111}
1112
1113
1114/**
1115 * Sets the permission bits for the specified MSR in the MSR bitmap.
1116 *
1117 * @param pVCpu Pointer to the VMCPU.
1118 * @param uMSR The MSR value.
1119 * @param enmRead Whether reading this MSR causes a VM-exit.
1120 * @param enmWrite Whether writing this MSR causes a VM-exit.
1121 */
1122static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1123{
1124 int32_t iBit;
1125 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1126
1127 /*
1128 * Layout:
1129 * 0x000 - 0x3ff - Low MSR read bits
1130 * 0x400 - 0x7ff - High MSR read bits
1131 * 0x800 - 0xbff - Low MSR write bits
1132 * 0xc00 - 0xfff - High MSR write bits
1133 */
1134 if (uMsr <= 0x00001FFF)
1135 iBit = uMsr;
1136 else if ( uMsr >= 0xC0000000
1137 && uMsr <= 0xC0001FFF)
1138 {
1139 iBit = (uMsr - 0xC0000000);
1140 pbMsrBitmap += 0x400;
1141 }
1142 else
1143 {
1144 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1145 return;
1146 }
1147
1148 Assert(iBit <= 0x1fff);
1149 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1150 ASMBitSet(pbMsrBitmap, iBit);
1151 else
1152 ASMBitClear(pbMsrBitmap, iBit);
1153
1154 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1155 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1156 else
1157 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1158}
1159
1160
1161#ifdef VBOX_STRICT
1162/**
1163 * Gets the permission bits for the specified MSR in the MSR bitmap.
1164 *
1165 * @returns VBox status code.
1166 * @retval VINF_SUCCESS if the specified MSR is found.
1167 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1168 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1169 *
1170 * @param pVCpu Pointer to the VMCPU.
1171 * @param uMsr The MSR.
1172 * @param penmRead Where to store the read permissions.
1173 * @param penmWrite Where to store the write permissions.
1174 */
1175static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1176{
1177 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1178 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1179 int32_t iBit;
1180 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1181
1182 /* See hmR0VmxSetMsrPermission() for the layout. */
1183 if (uMsr <= 0x00001FFF)
1184 iBit = uMsr;
1185 else if ( uMsr >= 0xC0000000
1186 && uMsr <= 0xC0001FFF)
1187 {
1188 iBit = (uMsr - 0xC0000000);
1189 pbMsrBitmap += 0x400;
1190 }
1191 else
1192 {
1193 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1194 return VERR_NOT_SUPPORTED;
1195 }
1196
1197 Assert(iBit <= 0x1fff);
1198 if (ASMBitTest(pbMsrBitmap, iBit))
1199 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1200 else
1201 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1202
1203 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1204 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1205 else
1206 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1207 return VINF_SUCCESS;
1208}
1209#endif /* VBOX_STRICT */
1210
1211
1212/**
1213 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1214 * area.
1215 *
1216 * @returns VBox status code.
1217 * @param pVCpu Pointer to the VMCPU.
1218 * @param cMsrs The number of MSRs.
1219 */
1220DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1221{
1222 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1223 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1224 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1225 {
1226 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1227 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1229 }
1230
1231 /* Update number of guest MSRs to load/store across the world-switch. */
1232 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1233 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1234
1235 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update the VCPU's copy of the MSR count. */
1239 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1240
1241 return VINF_SUCCESS;
1242}
1243
1244
1245/**
1246 * Adds a new (or updates the value of an existing) guest/host MSR
1247 * pair to be swapped during the world-switch as part of the
1248 * auto-load/store MSR area in the VMCS.
1249 *
1250 * @returns true if the MSR was added -and- its value was updated, false
1251 * otherwise.
1252 * @param pVCpu Pointer to the VMCPU.
1253 * @param uMsr The MSR.
1254 * @param uGuestMsr Value of the guest MSR.
1255 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1256 * necessary.
1257 */
1258static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cMsrs; i++)
1264 {
1265 if (pGuestMsr->u32Msr == uMsr)
1266 break;
1267 pGuestMsr++;
1268 }
1269
1270 bool fAdded = false;
1271 if (i == cMsrs)
1272 {
1273 ++cMsrs;
1274 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1275 AssertRC(rc);
1276
1277 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1278 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1279 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1280
1281 fAdded = true;
1282 }
1283
1284 /* Update the MSR values in the auto-load/store MSR area. */
1285 pGuestMsr->u32Msr = uMsr;
1286 pGuestMsr->u64Value = uGuestMsrValue;
1287
1288 /* Create/update the MSR slot in the host MSR area. */
1289 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1290 pHostMsr += i;
1291 pHostMsr->u32Msr = uMsr;
1292
1293 /*
1294 * Update the host MSR only when requested by the caller AND when we're
1295 * adding it to the auto-load/store area. Otherwise, it would have been
1296 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1297 */
1298 bool fUpdatedMsrValue = false;
1299 if ( fAdded
1300 && fUpdateHostMsr)
1301 {
1302 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1303 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1304 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1305 fUpdatedMsrValue = true;
1306 }
1307
1308 return fUpdatedMsrValue;
1309}
1310
1311
1312/**
1313 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1314 * auto-load/store MSR area in the VMCS.
1315 *
1316 * @returns VBox status code.
1317 * @param pVCpu Pointer to the VMCPU.
1318 * @param uMsr The MSR.
1319 */
1320static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1321{
1322 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1323 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1324 for (uint32_t i = 0; i < cMsrs; i++)
1325 {
1326 /* Find the MSR. */
1327 if (pGuestMsr->u32Msr == uMsr)
1328 {
1329 /* If it's the last MSR, simply reduce the count. */
1330 if (i == cMsrs - 1)
1331 {
1332 --cMsrs;
1333 break;
1334 }
1335
1336 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1337 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1338 pLastGuestMsr += cMsrs - 1;
1339 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1340 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1341
1342 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 pLastHostMsr += cMsrs - 1;
1345 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1346 pHostMsr->u64Value = pLastHostMsr->u64Value;
1347 --cMsrs;
1348 break;
1349 }
1350 pGuestMsr++;
1351 }
1352
1353 /* Update the VMCS if the count changed (meaning the MSR was found). */
1354 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1355 {
1356 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1357 AssertRCReturn(rc, rc);
1358
1359 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1360 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1361 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1362
1363 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1364 return VINF_SUCCESS;
1365 }
1366
1367 return VERR_NOT_FOUND;
1368}
1369
1370
1371/**
1372 * Checks if the specified guest MSR is part of the auto-load/store area in
1373 * the VMCS.
1374 *
1375 * @returns true if found, false otherwise.
1376 * @param pVCpu Pointer to the VMCPU.
1377 * @param uMsr The MSR to find.
1378 */
1379static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1380{
1381 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1383
1384 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1385 {
1386 if (pGuestMsr->u32Msr == uMsr)
1387 return true;
1388 }
1389 return false;
1390}
1391
1392
1393/**
1394 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1395 *
1396 * @param pVCpu Pointer to the VMCPU.
1397 *
1398 * @remarks No-long-jump zone!!!
1399 */
1400static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1401{
1402 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1403 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1405 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1406
1407 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1408 {
1409 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1410
1411 /*
1412 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1413 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1414 */
1415 if (pHostMsr->u32Msr == MSR_K6_EFER)
1416 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1417 else
1418 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1419 }
1420
1421 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1422}
1423
1424
1425#if HC_ARCH_BITS == 64
1426/**
1427 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1428 * perform lazy restoration of the host MSRs while leaving VT-x.
1429 *
1430 * @param pVCpu Pointer to the VMCPU.
1431 *
1432 * @remarks No-long-jump zone!!!
1433 */
1434static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1435{
1436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1437
1438 /*
1439 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1440 */
1441 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1442 {
1443 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1444 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1445 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1446 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1447 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1448 }
1449}
1450
1451
1452/**
1453 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1454 * lazily while leaving VT-x.
1455 *
1456 * @returns true if it does, false otherwise.
1457 * @param pVCpu Pointer to the VMCPU.
1458 * @param uMsr The MSR to check.
1459 */
1460static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1461{
1462 NOREF(pVCpu);
1463 switch (uMsr)
1464 {
1465 case MSR_K8_LSTAR:
1466 case MSR_K6_STAR:
1467 case MSR_K8_SF_MASK:
1468 case MSR_K8_KERNEL_GS_BASE:
1469 return true;
1470 }
1471 return false;
1472}
1473
1474
1475/**
1476 * Saves a set of guests MSRs back into the guest-CPU context.
1477 *
1478 * @param pVCpu Pointer to the VMCPU.
1479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1480 * out-of-sync. Make sure to update the required fields
1481 * before using them.
1482 *
1483 * @remarks No-long-jump zone!!!
1484 */
1485static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1486{
1487 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1488 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1489
1490 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1491 {
1492 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1493 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1494 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1495 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1496 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1497 }
1498}
1499
1500
1501/**
1502 * Loads a set of guests MSRs to allow read/passthru to the guest.
1503 *
1504 * The name of this function is slightly confusing. This function does NOT
1505 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1506 * common prefix for functions dealing with "lazy restoration" of the shared
1507 * MSRs.
1508 *
1509 * @param pVCpu Pointer to the VMCPU.
1510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1511 * out-of-sync. Make sure to update the required fields
1512 * before using them.
1513 *
1514 * @remarks No-long-jump zone!!!
1515 */
1516static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1517{
1518 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1520
1521 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1522 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1523 {
1524#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1525 do { \
1526 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1527 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1528 else \
1529 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1530 } while (0)
1531
1532 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1536#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1537 }
1538 else
1539 {
1540 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1541 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1542 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1543 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1544 }
1545 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1546}
1547
1548
1549/**
1550 * Performs lazy restoration of the set of host MSRs if they were previously
1551 * loaded with guest MSR values.
1552 *
1553 * @param pVCpu Pointer to the VMCPU.
1554 *
1555 * @remarks No-long-jump zone!!!
1556 * @remarks The guest MSRs should have been saved back into the guest-CPU
1557 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1558 */
1559static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1560{
1561 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1562 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1563
1564 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1565 {
1566 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1567 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1568 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1569 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1570 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1571 }
1572 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1573}
1574#endif /* HC_ARCH_BITS == 64 */
1575
1576
1577/**
1578 * Verifies that our cached values of the VMCS controls are all
1579 * consistent with what's actually present in the VMCS.
1580 *
1581 * @returns VBox status code.
1582 * @param pVCpu Pointer to the VMCPU.
1583 */
1584static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1585{
1586 uint32_t u32Val;
1587 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1588 AssertRCReturn(rc, rc);
1589 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1590 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1591
1592 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1595 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1600 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1605 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1610 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1611
1612 return VINF_SUCCESS;
1613}
1614
1615
1616#ifdef VBOX_STRICT
1617/**
1618 * Verifies that our cached host EFER value has not changed
1619 * since we cached it.
1620 *
1621 * @param pVCpu Pointer to the VMCPU.
1622 */
1623static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1624{
1625 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1626
1627 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1628 {
1629 uint64_t u64Val;
1630 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1631 AssertRC(rc);
1632
1633 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1634 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1635 }
1636}
1637
1638
1639/**
1640 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1641 * VMCS are correct.
1642 *
1643 * @param pVCpu Pointer to the VMCPU.
1644 */
1645static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1646{
1647 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1648
1649 /* Verify MSR counts in the VMCS are what we think it should be. */
1650 uint32_t cMsrs;
1651 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1652 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1653
1654 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1655 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1656
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1658 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1659
1660 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1661 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1662 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1663 {
1664 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1665 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1666 pGuestMsr->u32Msr, cMsrs));
1667
1668 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1669 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1670 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1671
1672 /* Verify that the permissions are as expected in the MSR bitmap. */
1673 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1674 {
1675 VMXMSREXITREAD enmRead;
1676 VMXMSREXITWRITE enmWrite;
1677 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1678 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1679 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1680 {
1681 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1682 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1683 }
1684 else
1685 {
1686 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1687 pGuestMsr->u32Msr, cMsrs));
1688 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1689 pGuestMsr->u32Msr, cMsrs));
1690 }
1691 }
1692 }
1693}
1694#endif /* VBOX_STRICT */
1695
1696
1697/**
1698 * Flushes the TLB using EPT.
1699 *
1700 * @returns VBox status code.
1701 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1702 * enmFlush).
1703 * @param enmFlush Type of flush.
1704 *
1705 * @remarks Caller is responsible for making sure this function is called only
1706 * when NestedPaging is supported and providing @a enmFlush that is
1707 * supported by the CPU.
1708 * @remarks Can be called with interrupts disabled.
1709 */
1710static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1711{
1712 uint64_t au64Descriptor[2];
1713 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1714 au64Descriptor[0] = 0;
1715 else
1716 {
1717 Assert(pVCpu);
1718 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1719 }
1720 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1721
1722 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1723 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1724 rc));
1725 if ( RT_SUCCESS(rc)
1726 && pVCpu)
1727 {
1728 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1729 }
1730}
1731
1732
1733/**
1734 * Flushes the TLB using VPID.
1735 *
1736 * @returns VBox status code.
1737 * @param pVM Pointer to the VM.
1738 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1739 * enmFlush).
1740 * @param enmFlush Type of flush.
1741 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1742 * on @a enmFlush).
1743 *
1744 * @remarks Can be called with interrupts disabled.
1745 */
1746static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1747{
1748 NOREF(pVM);
1749 AssertPtr(pVM);
1750 Assert(pVM->hm.s.vmx.fVpid);
1751
1752 uint64_t au64Descriptor[2];
1753 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1754 {
1755 au64Descriptor[0] = 0;
1756 au64Descriptor[1] = 0;
1757 }
1758 else
1759 {
1760 AssertPtr(pVCpu);
1761 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1762 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1763 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1764 au64Descriptor[1] = GCPtr;
1765 }
1766
1767 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1768 AssertMsg(rc == VINF_SUCCESS,
1769 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1770 if ( RT_SUCCESS(rc)
1771 && pVCpu)
1772 {
1773 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1774 }
1775}
1776
1777
1778/**
1779 * Invalidates a guest page by guest virtual address. Only relevant for
1780 * EPT/VPID, otherwise there is nothing really to invalidate.
1781 *
1782 * @returns VBox status code.
1783 * @param pVM Pointer to the VM.
1784 * @param pVCpu Pointer to the VMCPU.
1785 * @param GCVirt Guest virtual address of the page to invalidate.
1786 */
1787VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1788{
1789 AssertPtr(pVM);
1790 AssertPtr(pVCpu);
1791 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1792
1793 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1794 if (!fFlushPending)
1795 {
1796 /*
1797 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1798 * See @bugref{6043} and @bugref{6177}.
1799 *
1800 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1801 * function maybe called in a loop with individual addresses.
1802 */
1803 if (pVM->hm.s.vmx.fVpid)
1804 {
1805 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1806 {
1807 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1808 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1809 }
1810 else
1811 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1812 }
1813 else if (pVM->hm.s.fNestedPaging)
1814 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1815 }
1816
1817 return VINF_SUCCESS;
1818}
1819
1820
1821/**
1822 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1823 * otherwise there is nothing really to invalidate.
1824 *
1825 * @returns VBox status code.
1826 * @param pVM Pointer to the VM.
1827 * @param pVCpu Pointer to the VMCPU.
1828 * @param GCPhys Guest physical address of the page to invalidate.
1829 */
1830VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1831{
1832 NOREF(pVM); NOREF(GCPhys);
1833 LogFlowFunc(("%RGp\n", GCPhys));
1834
1835 /*
1836 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1837 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1838 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1839 */
1840 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1841 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1842 return VINF_SUCCESS;
1843}
1844
1845
1846/**
1847 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1848 * case where neither EPT nor VPID is supported by the CPU.
1849 *
1850 * @param pVM Pointer to the VM.
1851 * @param pVCpu Pointer to the VMCPU.
1852 * @param pCpu Pointer to the global HM struct.
1853 *
1854 * @remarks Called with interrupts disabled.
1855 */
1856static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1857{
1858 AssertPtr(pVCpu);
1859 AssertPtr(pCpu);
1860 NOREF(pVM);
1861
1862 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1863
1864 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1865#if 0
1866 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1867 pVCpu->hm.s.TlbShootdown.cPages = 0;
1868#endif
1869
1870 Assert(pCpu->idCpu != NIL_RTCPUID);
1871 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1872 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1873 pVCpu->hm.s.fForceTLBFlush = false;
1874 return;
1875}
1876
1877
1878/**
1879 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1880 *
1881 * @param pVM Pointer to the VM.
1882 * @param pVCpu Pointer to the VMCPU.
1883 * @param pCpu Pointer to the global HM CPU struct.
1884 * @remarks All references to "ASID" in this function pertains to "VPID" in
1885 * Intel's nomenclature. The reason is, to avoid confusion in compare
1886 * statements since the host-CPU copies are named "ASID".
1887 *
1888 * @remarks Called with interrupts disabled.
1889 */
1890static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1891{
1892#ifdef VBOX_WITH_STATISTICS
1893 bool fTlbFlushed = false;
1894# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1895# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1896 if (!fTlbFlushed) \
1897 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1898 } while (0)
1899#else
1900# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1901# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1902#endif
1903
1904 AssertPtr(pVM);
1905 AssertPtr(pCpu);
1906 AssertPtr(pVCpu);
1907 Assert(pCpu->idCpu != NIL_RTCPUID);
1908
1909 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1910 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1911 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1912
1913 /*
1914 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1915 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1916 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1917 */
1918 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1919 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1920 {
1921 ++pCpu->uCurrentAsid;
1922 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1923 {
1924 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1925 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1926 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1927 }
1928
1929 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1930 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932
1933 /*
1934 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1935 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1936 */
1937 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1939 HMVMX_SET_TAGGED_TLB_FLUSHED();
1940 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1941 }
1942
1943 /* Check for explicit TLB shootdowns. */
1944 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1945 {
1946 /*
1947 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1948 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1949 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1950 * but not guest-physical mappings.
1951 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1952 */
1953 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1955 HMVMX_SET_TAGGED_TLB_FLUSHED();
1956 }
1957
1958 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1959 * where it is commented out. Support individual entry flushing
1960 * someday. */
1961#if 0
1962 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1963 {
1964 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965
1966 /*
1967 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1968 * as supported by the CPU.
1969 */
1970 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1971 {
1972 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1973 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1974 }
1975 else
1976 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1977
1978 HMVMX_SET_TAGGED_TLB_FLUSHED();
1979 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980 pVCpu->hm.s.TlbShootdown.cPages = 0;
1981 }
1982#endif
1983
1984 pVCpu->hm.s.fForceTLBFlush = false;
1985
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVM Pointer to the VM.
2011 * @param pVCpu Pointer to the VMCPU.
2012 * @param pCpu Pointer to the global HM CPU struct.
2013 *
2014 * @remarks Called with interrupts disabled.
2015 */
2016static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2017{
2018 AssertPtr(pVM);
2019 AssertPtr(pVCpu);
2020 AssertPtr(pCpu);
2021 Assert(pCpu->idCpu != NIL_RTCPUID);
2022 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2023 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2024
2025 /*
2026 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2028 */
2029 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2030 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2031 {
2032 pVCpu->hm.s.fForceTLBFlush = true;
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 }
2035
2036 /* Check for explicit TLB shootdown flushes. */
2037 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2041 }
2042
2043 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2044 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2045
2046 if (pVCpu->hm.s.fForceTLBFlush)
2047 {
2048 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049 pVCpu->hm.s.fForceTLBFlush = false;
2050 }
2051 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2052 * where it is commented out. Support individual entry flushing
2053 * someday. */
2054#if 0
2055 else
2056 {
2057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2058 {
2059 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2060 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2061 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2062 }
2063 else
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2065
2066 pVCpu->hm.s.TlbShootdown.cPages = 0;
2067 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2068 }
2069#endif
2070}
2071
2072
2073/**
2074 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2075 *
2076 * @returns VBox status code.
2077 * @param pVM Pointer to the VM.
2078 * @param pVCpu Pointer to the VMCPU.
2079 * @param pCpu Pointer to the global HM CPU struct.
2080 *
2081 * @remarks Called with interrupts disabled.
2082 */
2083static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2084{
2085 AssertPtr(pVM);
2086 AssertPtr(pVCpu);
2087 AssertPtr(pCpu);
2088 Assert(pCpu->idCpu != NIL_RTCPUID);
2089 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2090 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2091
2092 /*
2093 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2094 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2095 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2096 */
2097 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099 {
2100 pVCpu->hm.s.fForceTLBFlush = true;
2101 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102 }
2103
2104 /* Check for explicit TLB shootdown flushes. */
2105 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106 {
2107 /*
2108 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2109 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2110 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2111 */
2112 pVCpu->hm.s.fForceTLBFlush = true;
2113 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2114 }
2115
2116 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2117 if (pVCpu->hm.s.fForceTLBFlush)
2118 {
2119 ++pCpu->uCurrentAsid;
2120 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2121 {
2122 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2123 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2124 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2125 }
2126
2127 pVCpu->hm.s.fForceTLBFlush = false;
2128 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2129 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2130 if (pCpu->fFlushAsidBeforeUse)
2131 {
2132 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2133 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2134 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2135 {
2136 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2137 pCpu->fFlushAsidBeforeUse = false;
2138 }
2139 else
2140 {
2141 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2142 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2143 }
2144 }
2145 }
2146 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2147 * where it is commented out. Support individual entry flushing
2148 * someday. */
2149#if 0
2150 else
2151 {
2152 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2153 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2154 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2155 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2156
2157 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2158 {
2159 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2160 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2161 {
2162 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2163 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2164 }
2165 else
2166 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2167
2168 pVCpu->hm.s.TlbShootdown.cPages = 0;
2169 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2170 }
2171 else
2172 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2173 }
2174#endif
2175
2176 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2177 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2178 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2179 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2180 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2181 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2182 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2183
2184 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2185 AssertRC(rc);
2186}
2187
2188
2189/**
2190 * Flushes the guest TLB entry based on CPU capabilities.
2191 *
2192 * @param pVCpu Pointer to the VMCPU.
2193 * @param pCpu Pointer to the global HM CPU struct.
2194 */
2195DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2196{
2197#ifdef HMVMX_ALWAYS_FLUSH_TLB
2198 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2199#endif
2200 PVM pVM = pVCpu->CTX_SUFF(pVM);
2201 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2202 {
2203 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2204 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2205 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2206 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2207 default:
2208 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2209 break;
2210 }
2211
2212 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2213 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2214
2215 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2216}
2217
2218
2219/**
2220 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2221 * TLB entries from the host TLB before VM-entry.
2222 *
2223 * @returns VBox status code.
2224 * @param pVM Pointer to the VM.
2225 */
2226static int hmR0VmxSetupTaggedTlb(PVM pVM)
2227{
2228 /*
2229 * Determine optimal flush type for Nested Paging.
2230 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2231 * guest execution (see hmR3InitFinalizeR0()).
2232 */
2233 if (pVM->hm.s.fNestedPaging)
2234 {
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2238 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2239 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2240 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2241 else
2242 {
2243 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2244 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2246 }
2247
2248 /* Make sure the write-back cacheable memory type for EPT is supported. */
2249 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2250 {
2251 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255 }
2256 else
2257 {
2258 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2259 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263
2264 /*
2265 * Determine optimal flush type for VPID.
2266 */
2267 if (pVM->hm.s.vmx.fVpid)
2268 {
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2270 {
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2272 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2273 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2274 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2275 else
2276 {
2277 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2279 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2280 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2281 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2283 pVM->hm.s.vmx.fVpid = false;
2284 }
2285 }
2286 else
2287 {
2288 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2289 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294
2295 /*
2296 * Setup the handler for flushing tagged-TLBs.
2297 */
2298 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2299 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2300 else if (pVM->hm.s.fNestedPaging)
2301 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2302 else if (pVM->hm.s.vmx.fVpid)
2303 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2304 else
2305 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2306 return VINF_SUCCESS;
2307}
2308
2309
2310/**
2311 * Sets up pin-based VM-execution controls in the VMCS.
2312 *
2313 * @returns VBox status code.
2314 * @param pVM Pointer to the VM.
2315 * @param pVCpu Pointer to the VMCPU.
2316 */
2317static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2318{
2319 AssertPtr(pVM);
2320 AssertPtr(pVCpu);
2321
2322 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2323 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2324
2325 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2326 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2327
2328 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2329 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2330
2331 /* Enable the VMX preemption timer. */
2332 if (pVM->hm.s.vmx.fUsePreemptTimer)
2333 {
2334 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2335 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2336 }
2337
2338 if ((val & zap) != val)
2339 {
2340 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2341 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2342 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2343 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2344 }
2345
2346 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2347 AssertRCReturn(rc, rc);
2348
2349 /* Update VCPU with the currently set pin-based VM-execution controls. */
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM Pointer to the VM.
2360 * @param pVMCPU Pointer to the VMCPU.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 }
2457
2458 /* If we're using virtual NMIs, we need the NMI-window exiting feature. */
2459 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2460 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
2461 {
2462 val |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
2463 }
2464
2465 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2466 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2467 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2468
2469 if ((val & zap) != val)
2470 {
2471 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2472 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2473 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2474 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2475 }
2476
2477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2478 AssertRCReturn(rc, rc);
2479
2480 /* Update VCPU with the currently set processor-based VM-execution controls. */
2481 pVCpu->hm.s.vmx.u32ProcCtls = val;
2482
2483 /*
2484 * Secondary processor-based VM-execution controls.
2485 */
2486 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2487 {
2488 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2489 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2490
2491 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2492 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2493
2494 if (pVM->hm.s.fNestedPaging)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2496 else
2497 {
2498 /*
2499 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2500 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2501 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2502 */
2503 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2505 }
2506
2507 if (pVM->hm.s.vmx.fVpid)
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2509
2510 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ((val & zap) != val)
2529 {
2530 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2531 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2532 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2533 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2534 }
2535
2536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2537 AssertRCReturn(rc, rc);
2538
2539 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2540 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2541 }
2542 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2543 {
2544 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2545 "available\n"));
2546 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2547 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2548 }
2549
2550 return VINF_SUCCESS;
2551}
2552
2553
2554/**
2555 * Sets up miscellaneous (everything other than Pin & Processor-based
2556 * VM-execution) control fields in the VMCS.
2557 *
2558 * @returns VBox status code.
2559 * @param pVM Pointer to the VM.
2560 * @param pVCpu Pointer to the VMCPU.
2561 */
2562static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2563{
2564 NOREF(pVM);
2565 AssertPtr(pVM);
2566 AssertPtr(pVCpu);
2567
2568 int rc = VERR_GENERAL_FAILURE;
2569
2570 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2571#if 0
2572 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2575
2576 /*
2577 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2578 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2579 * We thus use the exception bitmap to control it rather than use both.
2580 */
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2583
2584 /** @todo Explore possibility of using IO-bitmaps. */
2585 /* All IO & IOIO instructions cause VM-exits. */
2586 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2587 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2588
2589 /* Initialize the MSR-bitmap area. */
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2592 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2593#endif
2594
2595 /* Setup MSR auto-load/store area. */
2596 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2599 AssertRCReturn(rc, rc);
2600 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2601 AssertRCReturn(rc, rc);
2602
2603 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2604 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2605 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2606 AssertRCReturn(rc, rc);
2607
2608 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2610 AssertRCReturn(rc, rc);
2611
2612 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2613#if 0
2614 /* Setup debug controls */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2616 AssertRCReturn(rc, rc);
2617 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2618 AssertRCReturn(rc, rc);
2619#endif
2620
2621 return rc;
2622}
2623
2624
2625/**
2626 * Sets up the initial exception bitmap in the VMCS based on static conditions
2627 * (i.e. conditions that cannot ever change after starting the VM).
2628 *
2629 * @returns VBox status code.
2630 * @param pVM Pointer to the VM.
2631 * @param pVCpu Pointer to the VMCPU.
2632 */
2633static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2634{
2635 AssertPtr(pVM);
2636 AssertPtr(pVCpu);
2637
2638 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2639
2640 uint32_t u32XcptBitmap = 0;
2641
2642 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2643 if (!pVM->hm.s.fNestedPaging)
2644 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2645
2646 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2647 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2648 AssertRCReturn(rc, rc);
2649 return rc;
2650}
2651
2652
2653/**
2654 * Sets up the initial guest-state mask. The guest-state mask is consulted
2655 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2656 * for the nested virtualization case (as it would cause a VM-exit).
2657 *
2658 * @param pVCpu Pointer to the VMCPU.
2659 */
2660static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2661{
2662 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2663 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2664 return VINF_SUCCESS;
2665}
2666
2667
2668/**
2669 * Does per-VM VT-x initialization.
2670 *
2671 * @returns VBox status code.
2672 * @param pVM Pointer to the VM.
2673 */
2674VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2675{
2676 LogFlowFunc(("pVM=%p\n", pVM));
2677
2678 int rc = hmR0VmxStructsAlloc(pVM);
2679 if (RT_FAILURE(rc))
2680 {
2681 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2682 return rc;
2683 }
2684
2685 return VINF_SUCCESS;
2686}
2687
2688
2689/**
2690 * Does per-VM VT-x termination.
2691 *
2692 * @returns VBox status code.
2693 * @param pVM Pointer to the VM.
2694 */
2695VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2696{
2697 LogFlowFunc(("pVM=%p\n", pVM));
2698
2699#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2700 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2701 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2702#endif
2703 hmR0VmxStructsFree(pVM);
2704 return VINF_SUCCESS;
2705}
2706
2707
2708/**
2709 * Sets up the VM for execution under VT-x.
2710 * This function is only called once per-VM during initialization.
2711 *
2712 * @returns VBox status code.
2713 * @param pVM Pointer to the VM.
2714 */
2715VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2716{
2717 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2718 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2719
2720 LogFlowFunc(("pVM=%p\n", pVM));
2721
2722 /*
2723 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2724 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2725 */
2726 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2727 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2728 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2729 || !pVM->hm.s.vmx.pRealModeTSS))
2730 {
2731 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2732 return VERR_INTERNAL_ERROR;
2733 }
2734
2735#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2736 /*
2737 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2738 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2739 */
2740 if ( pVM->hm.s.fAllow64BitGuests
2741 && !HMVMX_IS_64BIT_HOST_MODE())
2742 {
2743 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2744 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2745 }
2746#endif
2747
2748 /* Initialize these always, see hmR3InitFinalizeR0().*/
2749 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2750 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2751
2752 /* Setup the tagged-TLB flush handlers. */
2753 int rc = hmR0VmxSetupTaggedTlb(pVM);
2754 if (RT_FAILURE(rc))
2755 {
2756 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2757 return rc;
2758 }
2759
2760 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2761 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2762#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2763 if ( HMVMX_IS_64BIT_HOST_MODE()
2764 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2765 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2767 {
2768 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2769 }
2770#endif
2771
2772 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2773 {
2774 PVMCPU pVCpu = &pVM->aCpus[i];
2775 AssertPtr(pVCpu);
2776 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2777
2778 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2779 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2780
2781 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2782 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2783 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2784
2785 /* Set revision dword at the beginning of the VMCS structure. */
2786 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2787
2788 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2789 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793 /* Load this VMCS as the current VMCS. */
2794 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2819 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2820 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2821 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2822#endif
2823
2824 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2825 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828
2829 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2830
2831 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2832 }
2833
2834 return VINF_SUCCESS;
2835}
2836
2837
2838/**
2839 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2840 * the VMCS.
2841 *
2842 * @returns VBox status code.
2843 * @param pVM Pointer to the VM.
2844 * @param pVCpu Pointer to the VMCPU.
2845 */
2846DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2847{
2848 NOREF(pVM); NOREF(pVCpu);
2849
2850 RTCCUINTREG uReg = ASMGetCR0();
2851 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2852 AssertRCReturn(rc, rc);
2853
2854#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2855 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2856 if (HMVMX_IS_64BIT_HOST_MODE())
2857 {
2858 uint64_t uRegCR3 = HMR0Get64bitCR3();
2859 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2860 }
2861 else
2862#endif
2863 {
2864 uReg = ASMGetCR3();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2866 }
2867 AssertRCReturn(rc, rc);
2868
2869 uReg = ASMGetCR4();
2870 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2871 AssertRCReturn(rc, rc);
2872 return rc;
2873}
2874
2875
2876#if HC_ARCH_BITS == 64
2877/**
2878 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2879 * requirements. See hmR0VmxSaveHostSegmentRegs().
2880 */
2881# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2882 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2883 { \
2884 bool fValidSelector = true; \
2885 if ((selValue) & X86_SEL_LDT) \
2886 { \
2887 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2888 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2889 } \
2890 if (fValidSelector) \
2891 { \
2892 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2893 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2894 } \
2895 (selValue) = 0; \
2896 }
2897#endif
2898
2899
2900/**
2901 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2902 * the host-state area in the VMCS.
2903 *
2904 * @returns VBox status code.
2905 * @param pVM Pointer to the VM.
2906 * @param pVCpu Pointer to the VMCPU.
2907 */
2908DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2909{
2910 NOREF(pVM);
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 */
2918 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2919 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2920#endif
2921
2922 /*
2923 * Host DS, ES, FS and GS segment registers.
2924 */
2925#if HC_ARCH_BITS == 64
2926 RTSEL uSelDS = ASMGetDS();
2927 RTSEL uSelES = ASMGetES();
2928 RTSEL uSelFS = ASMGetFS();
2929 RTSEL uSelGS = ASMGetGS();
2930#else
2931 RTSEL uSelDS = 0;
2932 RTSEL uSelES = 0;
2933 RTSEL uSelFS = 0;
2934 RTSEL uSelGS = 0;
2935#endif
2936
2937 /* Recalculate which host-state bits need to be manually restored. */
2938 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2944 RTSEL uSelCS;
2945 RTSEL uSelSS;
2946 if (HMVMX_IS_64BIT_HOST_MODE())
2947 {
2948 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2949 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2950 }
2951 else
2952 {
2953 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2954 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2955 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2956 }
2957#else
2958 RTSEL uSelCS = ASMGetCS();
2959 RTSEL uSelSS = ASMGetSS();
2960#endif
2961
2962 /*
2963 * Host TR segment register.
2964 */
2965 RTSEL uSelTR = ASMGetTR();
2966
2967#if HC_ARCH_BITS == 64
2968 /*
2969 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2970 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2971 */
2972 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2973 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2974 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2975 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2976# undef VMXLOCAL_ADJUST_HOST_SEG
2977#endif
2978
2979 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2980 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2981 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2982 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2983 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2984 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2985 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2986 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2987 Assert(uSelCS);
2988 Assert(uSelTR);
2989
2990 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2991#if 0
2992 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2993 Assert(uSelSS != 0);
2994#endif
2995
2996 /* Write these host selector fields into the host-state area in the VMCS. */
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2999#if HC_ARCH_BITS == 64
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
3001 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
3002 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
3003 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
3004#endif
3005 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3006
3007 /*
3008 * Host GDTR and IDTR.
3009 */
3010 RTGDTR Gdtr;
3011 RT_ZERO(Gdtr);
3012#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3013 if (HMVMX_IS_64BIT_HOST_MODE())
3014 {
3015 X86XDTR64 Gdtr64;
3016 X86XDTR64 Idtr64;
3017 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3018 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3019 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3020
3021 Gdtr.cbGdt = Gdtr64.cb;
3022 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3023 }
3024 else
3025#endif
3026 {
3027 RTIDTR Idtr;
3028 ASMGetGDTR(&Gdtr);
3029 ASMGetIDTR(&Idtr);
3030 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3031 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3032
3033#if HC_ARCH_BITS == 64
3034 /*
3035 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3036 * maximum limit (0xffff) on every VM-exit.
3037 */
3038 if (Gdtr.cbGdt != 0xffff)
3039 {
3040 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3041 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3042 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3043 }
3044
3045 /*
3046 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3047 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3048 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3049 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3050 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3051 * hosts where we are pretty sure it won't cause trouble.
3052 */
3053# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3054 if (Idtr.cbIdt < 0x0fff)
3055# else
3056 if (Idtr.cbIdt != 0xffff)
3057# endif
3058 {
3059 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3060 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3061 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3062 }
3063#endif
3064 }
3065
3066 /*
3067 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3068 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3069 */
3070 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3071 {
3072 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3073 return VERR_VMX_INVALID_HOST_STATE;
3074 }
3075
3076 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3077#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3078 if (HMVMX_IS_64BIT_HOST_MODE())
3079 {
3080 /* We need the 64-bit TR base for hybrid darwin. */
3081 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3082 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3083 }
3084 else
3085#endif
3086 {
3087 uintptr_t uTRBase;
3088#if HC_ARCH_BITS == 64
3089 uTRBase = X86DESC64_BASE(pDesc);
3090
3091 /*
3092 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3093 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3094 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3095 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3096 *
3097 * [1] See Intel spec. 3.5 "System Descriptor Types".
3098 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3099 */
3100 Assert(pDesc->System.u4Type == 11);
3101 if ( pDesc->System.u16LimitLow != 0x67
3102 || pDesc->System.u4LimitHigh)
3103 {
3104 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3105 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3106
3107 /* Store the GDTR here as we need it while restoring TR. */
3108 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3109 }
3110#else
3111 uTRBase = X86DESC_BASE(pDesc);
3112#endif
3113 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3114 }
3115 AssertRCReturn(rc, rc);
3116
3117 /*
3118 * Host FS base and GS base.
3119 */
3120#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3121 if (HMVMX_IS_64BIT_HOST_MODE())
3122 {
3123 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3124 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3125 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3126 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3127
3128# if HC_ARCH_BITS == 64
3129 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3130 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3131 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3132 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3133 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3134# endif
3135 }
3136#endif
3137 return rc;
3138}
3139
3140
3141/**
3142 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3143 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3144 * the host after every successful VM-exit.
3145 *
3146 * @returns VBox status code.
3147 * @param pVM Pointer to the VM.
3148 * @param pVCpu Pointer to the VMCPU.
3149 *
3150 * @remarks No-long-jump zone!!!
3151 */
3152DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3153{
3154 NOREF(pVM);
3155
3156 AssertPtr(pVCpu);
3157 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3158
3159 int rc = VINF_SUCCESS;
3160#if HC_ARCH_BITS == 64
3161 if (pVM->hm.s.fAllow64BitGuests)
3162 hmR0VmxLazySaveHostMsrs(pVCpu);
3163#endif
3164
3165 /*
3166 * Host Sysenter MSRs.
3167 */
3168 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3169 AssertRCReturn(rc, rc);
3170#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3171 if (HMVMX_IS_64BIT_HOST_MODE())
3172 {
3173 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3174 AssertRCReturn(rc, rc);
3175 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3176 }
3177 else
3178 {
3179 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3180 AssertRCReturn(rc, rc);
3181 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3182 }
3183#elif HC_ARCH_BITS == 32
3184 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3185 AssertRCReturn(rc, rc);
3186 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3187#else
3188 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3189 AssertRCReturn(rc, rc);
3190 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3191#endif
3192 AssertRCReturn(rc, rc);
3193
3194 /*
3195 * Host EFER MSR.
3196 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3197 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3198 */
3199 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3200 {
3201 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3202 AssertRCReturn(rc, rc);
3203 }
3204
3205 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3206 * hmR0VmxLoadGuestExitCtls() !! */
3207
3208 return rc;
3209}
3210
3211
3212/**
3213 * Figures out if we need to swap the EFER MSR which is
3214 * particularly expensive.
3215 *
3216 * We check all relevant bits. For now, that's everything
3217 * besides LMA/LME, as these two bits are handled by VM-entry,
3218 * see hmR0VmxLoadGuestExitCtls() and
3219 * hmR0VMxLoadGuestEntryCtls().
3220 *
3221 * @returns true if we need to load guest EFER, false otherwise.
3222 * @param pVCpu Pointer to the VMCPU.
3223 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3224 * out-of-sync. Make sure to update the required fields
3225 * before using them.
3226 *
3227 * @remarks Requires EFER, CR4.
3228 * @remarks No-long-jump zone!!!
3229 */
3230static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3231{
3232#ifdef HMVMX_ALWAYS_SWAP_EFER
3233 return true;
3234#endif
3235
3236#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3237 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3238 if (CPUMIsGuestInLongMode(pVCpu))
3239 return false;
3240#endif
3241
3242 PVM pVM = pVCpu->CTX_SUFF(pVM);
3243 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3244 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3245
3246 /*
3247 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3248 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3249 */
3250 if ( CPUMIsGuestInLongMode(pVCpu)
3251 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3252 {
3253 return true;
3254 }
3255
3256 /*
3257 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3258 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3259 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3260 */
3261 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3262 && (pMixedCtx->cr0 & X86_CR0_PG)
3263 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3264 {
3265 /* Assert that host is PAE capable. */
3266 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3267 return true;
3268 }
3269
3270 /** @todo Check the latest Intel spec. for any other bits,
3271 * like SMEP/SMAP? */
3272 return false;
3273}
3274
3275
3276/**
3277 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3278 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3279 * controls".
3280 *
3281 * @returns VBox status code.
3282 * @param pVCpu Pointer to the VMCPU.
3283 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3284 * out-of-sync. Make sure to update the required fields
3285 * before using them.
3286 *
3287 * @remarks Requires EFER.
3288 * @remarks No-long-jump zone!!!
3289 */
3290DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3291{
3292 int rc = VINF_SUCCESS;
3293 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3294 {
3295 PVM pVM = pVCpu->CTX_SUFF(pVM);
3296 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3297 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3298
3299 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3300 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3301
3302 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3303 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3304 {
3305 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3306 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3307 }
3308 else
3309 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3310
3311 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3312 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3313 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3314 {
3315 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3316 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3317 }
3318
3319 /*
3320 * The following should -not- be set (since we're not in SMM mode):
3321 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3322 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3323 */
3324
3325 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3326 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3327
3328 if ((val & zap) != val)
3329 {
3330 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3331 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3332 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3333 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3334 }
3335
3336 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3337 AssertRCReturn(rc, rc);
3338
3339 /* Update VCPU with the currently set VM-exit controls. */
3340 pVCpu->hm.s.vmx.u32EntryCtls = val;
3341 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3342 }
3343 return rc;
3344}
3345
3346
3347/**
3348 * Sets up the VM-exit controls in the VMCS.
3349 *
3350 * @returns VBox status code.
3351 * @param pVM Pointer to the VM.
3352 * @param pVCpu Pointer to the VMCPU.
3353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3354 * out-of-sync. Make sure to update the required fields
3355 * before using them.
3356 *
3357 * @remarks Requires EFER.
3358 */
3359DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3360{
3361 NOREF(pMixedCtx);
3362
3363 int rc = VINF_SUCCESS;
3364 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3365 {
3366 PVM pVM = pVCpu->CTX_SUFF(pVM);
3367 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3368 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3369
3370 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3371 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3372
3373 /*
3374 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3375 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3376 */
3377#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3378 if (HMVMX_IS_64BIT_HOST_MODE())
3379 {
3380 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3381 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3382 }
3383 else
3384 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3385#else
3386 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3387 {
3388 /* The switcher returns to long mode, EFER is managed by the switcher. */
3389 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3390 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3391 }
3392 else
3393 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3394#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3395
3396 /* If the newer VMCS fields for managing EFER exists, use it. */
3397 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3398 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3399 {
3400 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3401 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3402 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3403 }
3404
3405 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3406 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3407
3408 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3409 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3410 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3411
3412 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3413 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3414
3415 if ((val & zap) != val)
3416 {
3417 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3418 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3419 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3420 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3421 }
3422
3423 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3424 AssertRCReturn(rc, rc);
3425
3426 /* Update VCPU with the currently set VM-exit controls. */
3427 pVCpu->hm.s.vmx.u32ExitCtls = val;
3428 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3429 }
3430 return rc;
3431}
3432
3433
3434/**
3435 * Loads the guest APIC and related state.
3436 *
3437 * @returns VBox status code.
3438 * @param pVM Pointer to the VM.
3439 * @param pVCpu Pointer to the VMCPU.
3440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3441 * out-of-sync. Make sure to update the required fields
3442 * before using them.
3443 */
3444DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3445{
3446 NOREF(pMixedCtx);
3447
3448 int rc = VINF_SUCCESS;
3449 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3450 {
3451 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3452 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3453 {
3454 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3455
3456 bool fPendingIntr = false;
3457 uint8_t u8Tpr = 0;
3458 uint8_t u8PendingIntr = 0;
3459 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3460 AssertRCReturn(rc, rc);
3461
3462 /*
3463 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3464 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3465 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3466 * the interrupt when we VM-exit for other reasons.
3467 */
3468 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3469 uint32_t u32TprThreshold = 0;
3470 if (fPendingIntr)
3471 {
3472 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3473 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3474 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3475 if (u8PendingPriority <= u8TprPriority)
3476 u32TprThreshold = u8PendingPriority;
3477 else
3478 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3479 }
3480 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3481
3482 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3483 AssertRCReturn(rc, rc);
3484 }
3485
3486 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3487 }
3488 return rc;
3489}
3490
3491
3492/**
3493 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3494 *
3495 * @returns Guest's interruptibility-state.
3496 * @param pVCpu Pointer to the VMCPU.
3497 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3498 * out-of-sync. Make sure to update the required fields
3499 * before using them.
3500 *
3501 * @remarks No-long-jump zone!!!
3502 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3503 */
3504DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3505{
3506 /*
3507 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3508 * inhibit interrupts or clear any existing interrupt-inhibition.
3509 */
3510 uint32_t uIntrState = 0;
3511 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3512 {
3513 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3514 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3515 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3516 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3517 {
3518 /*
3519 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3520 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3521 */
3522 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3523 }
3524 else if (pMixedCtx->eflags.Bits.u1IF)
3525 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3526 else
3527 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3528 }
3529
3530 /*
3531 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3532 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3533 * setting this would block host-NMIs and IRET will not clear the blocking.
3534 *
3535 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3536 */
3537 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3538 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3539 {
3540 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3541 }
3542
3543 return uIntrState;
3544}
3545
3546
3547/**
3548 * Loads the guest's interruptibility-state into the guest-state area in the
3549 * VMCS.
3550 *
3551 * @returns VBox status code.
3552 * @param pVCpu Pointer to the VMCPU.
3553 * @param uIntrState The interruptibility-state to set.
3554 */
3555static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3556{
3557 NOREF(pVCpu);
3558 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3559 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3560 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3561 AssertRCReturn(rc, rc);
3562 return rc;
3563}
3564
3565
3566/**
3567 * Loads the guest's RIP into the guest-state area in the VMCS.
3568 *
3569 * @returns VBox status code.
3570 * @param pVCpu Pointer to the VMCPU.
3571 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3572 * out-of-sync. Make sure to update the required fields
3573 * before using them.
3574 *
3575 * @remarks No-long-jump zone!!!
3576 */
3577static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3578{
3579 int rc = VINF_SUCCESS;
3580 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3581 {
3582 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3583 AssertRCReturn(rc, rc);
3584
3585 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3586 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3587 HMCPU_CF_VALUE(pVCpu)));
3588 }
3589 return rc;
3590}
3591
3592
3593/**
3594 * Loads the guest's RSP into the guest-state area in the VMCS.
3595 *
3596 * @returns VBox status code.
3597 * @param pVCpu Pointer to the VMCPU.
3598 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3599 * out-of-sync. Make sure to update the required fields
3600 * before using them.
3601 *
3602 * @remarks No-long-jump zone!!!
3603 */
3604static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3605{
3606 int rc = VINF_SUCCESS;
3607 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3608 {
3609 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3610 AssertRCReturn(rc, rc);
3611
3612 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3613 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3614 }
3615 return rc;
3616}
3617
3618
3619/**
3620 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3621 *
3622 * @returns VBox status code.
3623 * @param pVCpu Pointer to the VMCPU.
3624 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3625 * out-of-sync. Make sure to update the required fields
3626 * before using them.
3627 *
3628 * @remarks No-long-jump zone!!!
3629 */
3630static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3631{
3632 int rc = VINF_SUCCESS;
3633 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3634 {
3635 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3636 Let us assert it as such and use 32-bit VMWRITE. */
3637 Assert(!(pMixedCtx->rflags.u64 >> 32));
3638 X86EFLAGS Eflags = pMixedCtx->eflags;
3639 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3640 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3641
3642 /*
3643 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3644 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3645 */
3646 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3647 {
3648 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3649 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3650 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3651 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3652 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3653 }
3654
3655 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3656 AssertRCReturn(rc, rc);
3657
3658 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3659 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3660 }
3661 return rc;
3662}
3663
3664
3665/**
3666 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3667 *
3668 * @returns VBox status code.
3669 * @param pVCpu Pointer to the VMCPU.
3670 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3671 * out-of-sync. Make sure to update the required fields
3672 * before using them.
3673 *
3674 * @remarks No-long-jump zone!!!
3675 */
3676DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3677{
3678 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3681 AssertRCReturn(rc, rc);
3682 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3683 AssertRCReturn(rc, rc);
3684 return rc;
3685}
3686
3687
3688/**
3689 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3690 * CR0 is partially shared with the host and we have to consider the FPU bits.
3691 *
3692 * @returns VBox status code.
3693 * @param pVM Pointer to the VM.
3694 * @param pVCpu Pointer to the VMCPU.
3695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3696 * out-of-sync. Make sure to update the required fields
3697 * before using them.
3698 *
3699 * @remarks No-long-jump zone!!!
3700 */
3701static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3702{
3703 /*
3704 * Guest CR0.
3705 * Guest FPU.
3706 */
3707 int rc = VINF_SUCCESS;
3708 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3709 {
3710 Assert(!(pMixedCtx->cr0 >> 32));
3711 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3712 PVM pVM = pVCpu->CTX_SUFF(pVM);
3713
3714 /* The guest's view (read access) of its CR0 is unblemished. */
3715 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3716 AssertRCReturn(rc, rc);
3717 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3718
3719 /* Setup VT-x's view of the guest CR0. */
3720 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3721 if (pVM->hm.s.fNestedPaging)
3722 {
3723 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3724 {
3725 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3726 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3727 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3728 }
3729 else
3730 {
3731 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3732 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3733 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734 }
3735
3736 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3737 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3738 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3739
3740 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3741 AssertRCReturn(rc, rc);
3742 }
3743 else
3744 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3745
3746 /*
3747 * Guest FPU bits.
3748 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3749 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3750 */
3751 u32GuestCR0 |= X86_CR0_NE;
3752 bool fInterceptNM = false;
3753 if (CPUMIsGuestFPUStateActive(pVCpu))
3754 {
3755 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3756 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3757 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3758 }
3759 else
3760 {
3761 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3762 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3763 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3764 }
3765
3766 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3767 bool fInterceptMF = false;
3768 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3769 fInterceptMF = true;
3770
3771 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3772 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3773 {
3774 Assert(PDMVmmDevHeapIsEnabled(pVM));
3775 Assert(pVM->hm.s.vmx.pRealModeTSS);
3776 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3777 fInterceptNM = true;
3778 fInterceptMF = true;
3779 }
3780 else
3781 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3782
3783 if (fInterceptNM)
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3785 else
3786 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3787
3788 if (fInterceptMF)
3789 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3790 else
3791 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3792
3793 /* Additional intercepts for debugging, define these yourself explicitly. */
3794#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3795 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3796 | RT_BIT(X86_XCPT_BP)
3797 | RT_BIT(X86_XCPT_DB)
3798 | RT_BIT(X86_XCPT_DE)
3799 | RT_BIT(X86_XCPT_NM)
3800 | RT_BIT(X86_XCPT_TS)
3801 | RT_BIT(X86_XCPT_UD)
3802 | RT_BIT(X86_XCPT_NP)
3803 | RT_BIT(X86_XCPT_SS)
3804 | RT_BIT(X86_XCPT_GP)
3805 | RT_BIT(X86_XCPT_PF)
3806 | RT_BIT(X86_XCPT_MF)
3807 ;
3808#elif defined(HMVMX_ALWAYS_TRAP_PF)
3809 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3810#endif
3811
3812 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3813
3814 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3815 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3816 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3817 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3818 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3819 else
3820 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3821
3822 u32GuestCR0 |= uSetCR0;
3823 u32GuestCR0 &= uZapCR0;
3824 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3825
3826 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3827 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3828 AssertRCReturn(rc, rc);
3829 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3830 AssertRCReturn(rc, rc);
3831 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3832 uZapCR0));
3833
3834 /*
3835 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3836 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3837 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3838 */
3839 uint32_t u32CR0Mask = 0;
3840 u32CR0Mask = X86_CR0_PE
3841 | X86_CR0_NE
3842 | X86_CR0_WP
3843 | X86_CR0_PG
3844 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3845 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3846 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3847
3848 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3849 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3850 * and @bugref{6944}. */
3851#if 0
3852 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3853 u32CR0Mask &= ~X86_CR0_PE;
3854#endif
3855 if (pVM->hm.s.fNestedPaging)
3856 u32CR0Mask &= ~X86_CR0_WP;
3857
3858 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3859 if (fInterceptNM)
3860 {
3861 u32CR0Mask |= X86_CR0_TS
3862 | X86_CR0_MP;
3863 }
3864
3865 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3866 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3867 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3868 AssertRCReturn(rc, rc);
3869 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3870
3871 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3872 }
3873 return rc;
3874}
3875
3876
3877/**
3878 * Loads the guest control registers (CR3, CR4) into the guest-state area
3879 * in the VMCS.
3880 *
3881 * @returns VBox status code.
3882 * @param pVM Pointer to the VM.
3883 * @param pVCpu Pointer to the VMCPU.
3884 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3885 * out-of-sync. Make sure to update the required fields
3886 * before using them.
3887 *
3888 * @remarks No-long-jump zone!!!
3889 */
3890static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3891{
3892 int rc = VINF_SUCCESS;
3893 PVM pVM = pVCpu->CTX_SUFF(pVM);
3894
3895 /*
3896 * Guest CR2.
3897 * It's always loaded in the assembler code. Nothing to do here.
3898 */
3899
3900 /*
3901 * Guest CR3.
3902 */
3903 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3904 {
3905 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3906 if (pVM->hm.s.fNestedPaging)
3907 {
3908 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3909
3910 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3911 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3913 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3914
3915 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3916 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3917 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3918
3919 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3920 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3921 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3922 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3923
3924 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3925 AssertRCReturn(rc, rc);
3926 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3927
3928 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3929 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3930 {
3931 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3932 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3933 {
3934 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3935 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3936 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3937 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3939 }
3940
3941 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3942 have Unrestricted Execution to handle the guest when it's not using paging. */
3943 GCPhysGuestCR3 = pMixedCtx->cr3;
3944 }
3945 else
3946 {
3947 /*
3948 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3949 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3950 * EPT takes care of translating it to host-physical addresses.
3951 */
3952 RTGCPHYS GCPhys;
3953 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3954 Assert(PDMVmmDevHeapIsEnabled(pVM));
3955
3956 /* We obtain it here every time as the guest could have relocated this PCI region. */
3957 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3958 AssertRCReturn(rc, rc);
3959
3960 GCPhysGuestCR3 = GCPhys;
3961 }
3962
3963 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3964 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3965 }
3966 else
3967 {
3968 /* Non-nested paging case, just use the hypervisor's CR3. */
3969 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3970
3971 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3972 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3973 }
3974 AssertRCReturn(rc, rc);
3975
3976 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3977 }
3978
3979 /*
3980 * Guest CR4.
3981 */
3982 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3983 {
3984 Assert(!(pMixedCtx->cr4 >> 32));
3985 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3986
3987 /* The guest's view of its CR4 is unblemished. */
3988 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3989 AssertRCReturn(rc, rc);
3990 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3991
3992 /* Setup VT-x's view of the guest CR4. */
3993 /*
3994 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3995 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3996 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3997 */
3998 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3999 {
4000 Assert(pVM->hm.s.vmx.pRealModeTSS);
4001 Assert(PDMVmmDevHeapIsEnabled(pVM));
4002 u32GuestCR4 &= ~X86_CR4_VME;
4003 }
4004
4005 if (pVM->hm.s.fNestedPaging)
4006 {
4007 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4008 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4009 {
4010 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4011 u32GuestCR4 |= X86_CR4_PSE;
4012 /* Our identity mapping is a 32-bit page directory. */
4013 u32GuestCR4 &= ~X86_CR4_PAE;
4014 }
4015 /* else use guest CR4.*/
4016 }
4017 else
4018 {
4019 /*
4020 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4021 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4022 */
4023 switch (pVCpu->hm.s.enmShadowMode)
4024 {
4025 case PGMMODE_REAL: /* Real-mode. */
4026 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4027 case PGMMODE_32_BIT: /* 32-bit paging. */
4028 {
4029 u32GuestCR4 &= ~X86_CR4_PAE;
4030 break;
4031 }
4032
4033 case PGMMODE_PAE: /* PAE paging. */
4034 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4035 {
4036 u32GuestCR4 |= X86_CR4_PAE;
4037 break;
4038 }
4039
4040 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4041 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4042#ifdef VBOX_ENABLE_64_BITS_GUESTS
4043 break;
4044#endif
4045 default:
4046 AssertFailed();
4047 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4048 }
4049 }
4050
4051 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4052 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4053 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4054 u32GuestCR4 |= uSetCR4;
4055 u32GuestCR4 &= uZapCR4;
4056
4057 /* Write VT-x's view of the guest CR4 into the VMCS. */
4058 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4059 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4060 AssertRCReturn(rc, rc);
4061
4062 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4063 uint32_t u32CR4Mask = 0;
4064 u32CR4Mask = X86_CR4_VME
4065 | X86_CR4_PAE
4066 | X86_CR4_PGE
4067 | X86_CR4_PSE
4068 | X86_CR4_VMXE;
4069 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4070 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4071 AssertRCReturn(rc, rc);
4072
4073 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4074 }
4075 return rc;
4076}
4077
4078
4079/**
4080 * Loads the guest debug registers into the guest-state area in the VMCS.
4081 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4082 *
4083 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4084 *
4085 * @returns VBox status code.
4086 * @param pVCpu Pointer to the VMCPU.
4087 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4088 * out-of-sync. Make sure to update the required fields
4089 * before using them.
4090 *
4091 * @remarks No-long-jump zone!!!
4092 */
4093static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4094{
4095 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4096 return VINF_SUCCESS;
4097
4098#ifdef VBOX_STRICT
4099 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4100 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4101 {
4102 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4103 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4104 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4105 }
4106#endif
4107
4108 int rc;
4109 PVM pVM = pVCpu->CTX_SUFF(pVM);
4110 bool fInterceptDB = false;
4111 bool fInterceptMovDRx = false;
4112 if ( pVCpu->hm.s.fSingleInstruction
4113 || DBGFIsStepping(pVCpu))
4114 {
4115 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4116 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4117 {
4118 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4119 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4120 AssertRCReturn(rc, rc);
4121 Assert(fInterceptDB == false);
4122 }
4123 else
4124 {
4125 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4126 pVCpu->hm.s.fClearTrapFlag = true;
4127 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4128 fInterceptDB = true;
4129 }
4130 }
4131
4132 if ( fInterceptDB
4133 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4134 {
4135 /*
4136 * Use the combined guest and host DRx values found in the hypervisor
4137 * register set because the debugger has breakpoints active or someone
4138 * is single stepping on the host side without a monitor trap flag.
4139 *
4140 * Note! DBGF expects a clean DR6 state before executing guest code.
4141 */
4142#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4143 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4144 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4145 {
4146 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4147 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4148 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4149 }
4150 else
4151#endif
4152 if (!CPUMIsHyperDebugStateActive(pVCpu))
4153 {
4154 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4155 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4156 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4157 }
4158
4159 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4160 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4161 AssertRCReturn(rc, rc);
4162
4163 pVCpu->hm.s.fUsingHyperDR7 = true;
4164 fInterceptDB = true;
4165 fInterceptMovDRx = true;
4166 }
4167 else
4168 {
4169 /*
4170 * If the guest has enabled debug registers, we need to load them prior to
4171 * executing guest code so they'll trigger at the right time.
4172 */
4173 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4174 {
4175#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4176 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4177 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4178 {
4179 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4180 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4181 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4182 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4183 }
4184 else
4185#endif
4186 if (!CPUMIsGuestDebugStateActive(pVCpu))
4187 {
4188 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4189 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4190 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4191 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4192 }
4193 Assert(!fInterceptDB);
4194 Assert(!fInterceptMovDRx);
4195 }
4196 /*
4197 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4198 * must intercept #DB in order to maintain a correct DR6 guest value.
4199 */
4200#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4201 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4202 && !CPUMIsGuestDebugStateActive(pVCpu))
4203#else
4204 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4205#endif
4206 {
4207 fInterceptMovDRx = true;
4208 fInterceptDB = true;
4209 }
4210
4211 /* Update guest DR7. */
4212 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4213 AssertRCReturn(rc, rc);
4214
4215 pVCpu->hm.s.fUsingHyperDR7 = false;
4216 }
4217
4218 /*
4219 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4220 */
4221 if (fInterceptDB)
4222 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4223 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4224 {
4225#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4226 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4227#endif
4228 }
4229 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4230 AssertRCReturn(rc, rc);
4231
4232 /*
4233 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4234 */
4235 if (fInterceptMovDRx)
4236 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4237 else
4238 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4239 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4240 AssertRCReturn(rc, rc);
4241
4242 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4243 return VINF_SUCCESS;
4244}
4245
4246
4247#ifdef VBOX_STRICT
4248/**
4249 * Strict function to validate segment registers.
4250 *
4251 * @remarks ASSUMES CR0 is up to date.
4252 */
4253static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4254{
4255 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4256 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4257 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4258 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4259 && ( !CPUMIsGuestInRealModeEx(pCtx)
4260 && !CPUMIsGuestInV86ModeEx(pCtx)))
4261 {
4262 /* Protected mode checks */
4263 /* CS */
4264 Assert(pCtx->cs.Attr.n.u1Present);
4265 Assert(!(pCtx->cs.Attr.u & 0xf00));
4266 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4267 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4268 || !(pCtx->cs.Attr.n.u1Granularity));
4269 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4270 || (pCtx->cs.Attr.n.u1Granularity));
4271 /* CS cannot be loaded with NULL in protected mode. */
4272 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4273 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4274 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4275 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4276 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4277 else
4278 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4279 /* SS */
4280 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4281 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4282 if ( !(pCtx->cr0 & X86_CR0_PE)
4283 || pCtx->cs.Attr.n.u4Type == 3)
4284 {
4285 Assert(!pCtx->ss.Attr.n.u2Dpl);
4286 }
4287 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4288 {
4289 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4290 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4291 Assert(pCtx->ss.Attr.n.u1Present);
4292 Assert(!(pCtx->ss.Attr.u & 0xf00));
4293 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4294 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4295 || !(pCtx->ss.Attr.n.u1Granularity));
4296 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4297 || (pCtx->ss.Attr.n.u1Granularity));
4298 }
4299 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4300 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4301 {
4302 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4303 Assert(pCtx->ds.Attr.n.u1Present);
4304 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4305 Assert(!(pCtx->ds.Attr.u & 0xf00));
4306 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4307 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4308 || !(pCtx->ds.Attr.n.u1Granularity));
4309 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4310 || (pCtx->ds.Attr.n.u1Granularity));
4311 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4312 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4313 }
4314 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4315 {
4316 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4317 Assert(pCtx->es.Attr.n.u1Present);
4318 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4319 Assert(!(pCtx->es.Attr.u & 0xf00));
4320 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4321 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4322 || !(pCtx->es.Attr.n.u1Granularity));
4323 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4324 || (pCtx->es.Attr.n.u1Granularity));
4325 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4326 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4327 }
4328 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4329 {
4330 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4331 Assert(pCtx->fs.Attr.n.u1Present);
4332 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4333 Assert(!(pCtx->fs.Attr.u & 0xf00));
4334 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4335 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4336 || !(pCtx->fs.Attr.n.u1Granularity));
4337 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4338 || (pCtx->fs.Attr.n.u1Granularity));
4339 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4340 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4341 }
4342 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4343 {
4344 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4345 Assert(pCtx->gs.Attr.n.u1Present);
4346 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4347 Assert(!(pCtx->gs.Attr.u & 0xf00));
4348 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4349 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4350 || !(pCtx->gs.Attr.n.u1Granularity));
4351 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4352 || (pCtx->gs.Attr.n.u1Granularity));
4353 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4354 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4355 }
4356 /* 64-bit capable CPUs. */
4357# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4358 if (HMVMX_IS_64BIT_HOST_MODE())
4359 {
4360 Assert(!(pCtx->cs.u64Base >> 32));
4361 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4362 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4363 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4364 }
4365# endif
4366 }
4367 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4368 || ( CPUMIsGuestInRealModeEx(pCtx)
4369 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4370 {
4371 /* Real and v86 mode checks. */
4372 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4373 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4374 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4375 {
4376 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4377 }
4378 else
4379 {
4380 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4381 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4382 }
4383
4384 /* CS */
4385 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4386 Assert(pCtx->cs.u32Limit == 0xffff);
4387 Assert(u32CSAttr == 0xf3);
4388 /* SS */
4389 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4390 Assert(pCtx->ss.u32Limit == 0xffff);
4391 Assert(u32SSAttr == 0xf3);
4392 /* DS */
4393 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4394 Assert(pCtx->ds.u32Limit == 0xffff);
4395 Assert(u32DSAttr == 0xf3);
4396 /* ES */
4397 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4398 Assert(pCtx->es.u32Limit == 0xffff);
4399 Assert(u32ESAttr == 0xf3);
4400 /* FS */
4401 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4402 Assert(pCtx->fs.u32Limit == 0xffff);
4403 Assert(u32FSAttr == 0xf3);
4404 /* GS */
4405 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4406 Assert(pCtx->gs.u32Limit == 0xffff);
4407 Assert(u32GSAttr == 0xf3);
4408 /* 64-bit capable CPUs. */
4409# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4410 if (HMVMX_IS_64BIT_HOST_MODE())
4411 {
4412 Assert(!(pCtx->cs.u64Base >> 32));
4413 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4414 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4415 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4416 }
4417# endif
4418 }
4419}
4420#endif /* VBOX_STRICT */
4421
4422
4423/**
4424 * Writes a guest segment register into the guest-state area in the VMCS.
4425 *
4426 * @returns VBox status code.
4427 * @param pVCpu Pointer to the VMCPU.
4428 * @param idxSel Index of the selector in the VMCS.
4429 * @param idxLimit Index of the segment limit in the VMCS.
4430 * @param idxBase Index of the segment base in the VMCS.
4431 * @param idxAccess Index of the access rights of the segment in the VMCS.
4432 * @param pSelReg Pointer to the segment selector.
4433 *
4434 * @remarks No-long-jump zone!!!
4435 */
4436static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4437 uint32_t idxAccess, PCPUMSELREG pSelReg)
4438{
4439 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4440 AssertRCReturn(rc, rc);
4441 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4442 AssertRCReturn(rc, rc);
4443 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4444 AssertRCReturn(rc, rc);
4445
4446 uint32_t u32Access = pSelReg->Attr.u;
4447 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4448 {
4449 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4450 u32Access = 0xf3;
4451 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4452 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4453 }
4454 else
4455 {
4456 /*
4457 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4458 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4459 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4460 * loaded in protected-mode have their attribute as 0.
4461 */
4462 if (!u32Access)
4463 u32Access = X86DESCATTR_UNUSABLE;
4464 }
4465
4466 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4467 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4468 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4469
4470 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4471 AssertRCReturn(rc, rc);
4472 return rc;
4473}
4474
4475
4476/**
4477 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4478 * into the guest-state area in the VMCS.
4479 *
4480 * @returns VBox status code.
4481 * @param pVM Pointer to the VM.
4482 * @param pVCPU Pointer to the VMCPU.
4483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4484 * out-of-sync. Make sure to update the required fields
4485 * before using them.
4486 *
4487 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4488 * @remarks No-long-jump zone!!!
4489 */
4490static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4491{
4492 int rc = VERR_INTERNAL_ERROR_5;
4493 PVM pVM = pVCpu->CTX_SUFF(pVM);
4494
4495 /*
4496 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4497 */
4498 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4499 {
4500 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4501 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4502 {
4503 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4504 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4505 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4506 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4507 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4508 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4509 }
4510
4511#ifdef VBOX_WITH_REM
4512 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4513 {
4514 Assert(pVM->hm.s.vmx.pRealModeTSS);
4515 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4516 if ( pVCpu->hm.s.vmx.fWasInRealMode
4517 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4518 {
4519 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4520 in real-mode (e.g. OpenBSD 4.0) */
4521 REMFlushTBs(pVM);
4522 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4523 pVCpu->hm.s.vmx.fWasInRealMode = false;
4524 }
4525 }
4526#endif
4527 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4528 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4529 AssertRCReturn(rc, rc);
4530 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4531 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4532 AssertRCReturn(rc, rc);
4533 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4534 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4535 AssertRCReturn(rc, rc);
4536 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4537 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4538 AssertRCReturn(rc, rc);
4539 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4540 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4541 AssertRCReturn(rc, rc);
4542 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4543 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4544 AssertRCReturn(rc, rc);
4545
4546#ifdef VBOX_STRICT
4547 /* Validate. */
4548 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4549#endif
4550
4551 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4552 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4553 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4554 }
4555
4556 /*
4557 * Guest TR.
4558 */
4559 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4560 {
4561 /*
4562 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4563 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4564 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4565 */
4566 uint16_t u16Sel = 0;
4567 uint32_t u32Limit = 0;
4568 uint64_t u64Base = 0;
4569 uint32_t u32AccessRights = 0;
4570
4571 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4572 {
4573 u16Sel = pMixedCtx->tr.Sel;
4574 u32Limit = pMixedCtx->tr.u32Limit;
4575 u64Base = pMixedCtx->tr.u64Base;
4576 u32AccessRights = pMixedCtx->tr.Attr.u;
4577 }
4578 else
4579 {
4580 Assert(pVM->hm.s.vmx.pRealModeTSS);
4581 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4582
4583 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4584 RTGCPHYS GCPhys;
4585 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4586 AssertRCReturn(rc, rc);
4587
4588 X86DESCATTR DescAttr;
4589 DescAttr.u = 0;
4590 DescAttr.n.u1Present = 1;
4591 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4592
4593 u16Sel = 0;
4594 u32Limit = HM_VTX_TSS_SIZE;
4595 u64Base = GCPhys; /* in real-mode phys = virt. */
4596 u32AccessRights = DescAttr.u;
4597 }
4598
4599 /* Validate. */
4600 Assert(!(u16Sel & RT_BIT(2)));
4601 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4602 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4603 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4604 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4605 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4606 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4607 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4608 Assert( (u32Limit & 0xfff) == 0xfff
4609 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4610 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4611 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4612
4613 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4614 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4615 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4616 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4617
4618 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4619 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4620 }
4621
4622 /*
4623 * Guest GDTR.
4624 */
4625 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4626 {
4627 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4628 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4629
4630 /* Validate. */
4631 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4632
4633 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4634 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4635 }
4636
4637 /*
4638 * Guest LDTR.
4639 */
4640 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4641 {
4642 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4643 uint32_t u32Access = 0;
4644 if (!pMixedCtx->ldtr.Attr.u)
4645 u32Access = X86DESCATTR_UNUSABLE;
4646 else
4647 u32Access = pMixedCtx->ldtr.Attr.u;
4648
4649 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4650 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4651 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4652 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4653
4654 /* Validate. */
4655 if (!(u32Access & X86DESCATTR_UNUSABLE))
4656 {
4657 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4658 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4659 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4660 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4661 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4662 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4663 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4664 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4665 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4666 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4667 }
4668
4669 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4670 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4671 }
4672
4673 /*
4674 * Guest IDTR.
4675 */
4676 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4677 {
4678 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4679 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4680
4681 /* Validate. */
4682 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4683
4684 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4685 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4686 }
4687
4688 return VINF_SUCCESS;
4689}
4690
4691
4692/**
4693 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4694 * areas. These MSRs will automatically be loaded to the host CPU on every
4695 * successful VM entry and stored from the host CPU on every successful VM-exit.
4696 *
4697 * This also creates/updates MSR slots for the host MSRs. The actual host
4698 * MSR values are -not- updated here for performance reasons. See
4699 * hmR0VmxSaveHostMsrs().
4700 *
4701 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4702 *
4703 * @returns VBox status code.
4704 * @param pVCpu Pointer to the VMCPU.
4705 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4706 * out-of-sync. Make sure to update the required fields
4707 * before using them.
4708 *
4709 * @remarks No-long-jump zone!!!
4710 */
4711static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4712{
4713 AssertPtr(pVCpu);
4714 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4715
4716 /*
4717 * MSRs that we use the auto-load/store MSR area in the VMCS.
4718 */
4719 PVM pVM = pVCpu->CTX_SUFF(pVM);
4720 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4721 {
4722 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4723#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4724 if (pVM->hm.s.fAllow64BitGuests)
4725 {
4726 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4727 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4728 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4729 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4730# ifdef DEBUG
4731 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4732 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4733 {
4734 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4735 pMsr->u64Value));
4736 }
4737# endif
4738 }
4739#endif
4740 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4741 }
4742
4743 /*
4744 * Guest Sysenter MSRs.
4745 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4746 * VM-exits on WRMSRs for these MSRs.
4747 */
4748 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4749 {
4750 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4751 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4752 }
4753
4754 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4755 {
4756 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4757 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4758 }
4759
4760 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4761 {
4762 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4763 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4764 }
4765
4766 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4767 {
4768 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4769 {
4770 /*
4771 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4772 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4773 */
4774 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4775 {
4776 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4777 AssertRCReturn(rc,rc);
4778 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4779 }
4780 else
4781 {
4782 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4783 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4784 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4785 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4786 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4787 }
4788 }
4789 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4790 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4791 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4792 }
4793
4794 return VINF_SUCCESS;
4795}
4796
4797
4798/**
4799 * Loads the guest activity state into the guest-state area in the VMCS.
4800 *
4801 * @returns VBox status code.
4802 * @param pVCpu Pointer to the VMCPU.
4803 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4804 * out-of-sync. Make sure to update the required fields
4805 * before using them.
4806 *
4807 * @remarks No-long-jump zone!!!
4808 */
4809static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4810{
4811 NOREF(pCtx);
4812 /** @todo See if we can make use of other states, e.g.
4813 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4814 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4815 {
4816 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4817 AssertRCReturn(rc, rc);
4818
4819 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4820 }
4821 return VINF_SUCCESS;
4822}
4823
4824
4825/**
4826 * Sets up the appropriate function to run guest code.
4827 *
4828 * @returns VBox status code.
4829 * @param pVCpu Pointer to the VMCPU.
4830 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4831 * out-of-sync. Make sure to update the required fields
4832 * before using them.
4833 *
4834 * @remarks No-long-jump zone!!!
4835 */
4836static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4837{
4838 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4839 {
4840#ifndef VBOX_ENABLE_64_BITS_GUESTS
4841 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4842#endif
4843 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4844#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4845 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4846 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4847 {
4848 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4849 {
4850 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4851 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4852 | HM_CHANGED_VMX_EXIT_CTLS
4853 | HM_CHANGED_VMX_ENTRY_CTLS
4854 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4855 }
4856 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4857 }
4858#else
4859 /* 64-bit host or hybrid host. */
4860 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4861#endif
4862 }
4863 else
4864 {
4865 /* Guest is not in long mode, use the 32-bit handler. */
4866#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4867 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4868 {
4869 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4870 {
4871 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4872 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4873 | HM_CHANGED_VMX_EXIT_CTLS
4874 | HM_CHANGED_VMX_ENTRY_CTLS
4875 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4876 }
4877 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4878 }
4879#else
4880 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4881#endif
4882 }
4883 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4884 return VINF_SUCCESS;
4885}
4886
4887
4888/**
4889 * Wrapper for running the guest code in VT-x.
4890 *
4891 * @returns VBox strict status code.
4892 * @param pVM Pointer to the VM.
4893 * @param pVCpu Pointer to the VMCPU.
4894 * @param pCtx Pointer to the guest-CPU context.
4895 *
4896 * @remarks No-long-jump zone!!!
4897 */
4898DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4899{
4900 /*
4901 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4902 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4903 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4904 */
4905 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4906 /** @todo Add stats for resume vs launch. */
4907#ifdef VBOX_WITH_KERNEL_USING_XMM
4908 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4909#else
4910 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4911#endif
4912}
4913
4914
4915/**
4916 * Reports world-switch error and dumps some useful debug info.
4917 *
4918 * @param pVM Pointer to the VM.
4919 * @param pVCpu Pointer to the VMCPU.
4920 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4921 * @param pCtx Pointer to the guest-CPU context.
4922 * @param pVmxTransient Pointer to the VMX transient structure (only
4923 * exitReason updated).
4924 */
4925static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4926{
4927 Assert(pVM);
4928 Assert(pVCpu);
4929 Assert(pCtx);
4930 Assert(pVmxTransient);
4931 HMVMX_ASSERT_PREEMPT_SAFE();
4932
4933 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4934 switch (rcVMRun)
4935 {
4936 case VERR_VMX_INVALID_VMXON_PTR:
4937 AssertFailed();
4938 break;
4939 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4940 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4941 {
4942 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4943 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4944 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4945 AssertRC(rc);
4946
4947 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4948 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4949 Cannot do it here as we may have been long preempted. */
4950
4951#ifdef VBOX_STRICT
4952 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4953 pVmxTransient->uExitReason));
4954 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4955 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4956 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4957 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4958 else
4959 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4960 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4961 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4962
4963 /* VMX control bits. */
4964 uint32_t u32Val;
4965 uint64_t u64Val;
4966 HMVMXHCUINTREG uHCReg;
4967 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4968 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4969 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4970 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4971 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4972 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4989 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4990 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4991 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4992 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4993 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4994 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4999 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5000 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5001 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5002 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5003 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5004 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5005 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5006 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5007 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5008 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5009
5010 /* Guest bits. */
5011 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5012 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5013 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5014 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5015 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5016 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5017 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5018 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5019
5020 /* Host bits. */
5021 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5022 Log4(("Host CR0 %#RHr\n", uHCReg));
5023 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5024 Log4(("Host CR3 %#RHr\n", uHCReg));
5025 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5026 Log4(("Host CR4 %#RHr\n", uHCReg));
5027
5028 RTGDTR HostGdtr;
5029 PCX86DESCHC pDesc;
5030 ASMGetGDTR(&HostGdtr);
5031 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5032 Log4(("Host CS %#08x\n", u32Val));
5033 if (u32Val < HostGdtr.cbGdt)
5034 {
5035 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5036 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5037 }
5038
5039 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5040 Log4(("Host DS %#08x\n", u32Val));
5041 if (u32Val < HostGdtr.cbGdt)
5042 {
5043 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5044 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5045 }
5046
5047 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5048 Log4(("Host ES %#08x\n", u32Val));
5049 if (u32Val < HostGdtr.cbGdt)
5050 {
5051 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5052 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5053 }
5054
5055 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5056 Log4(("Host FS %#08x\n", u32Val));
5057 if (u32Val < HostGdtr.cbGdt)
5058 {
5059 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5060 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5061 }
5062
5063 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5064 Log4(("Host GS %#08x\n", u32Val));
5065 if (u32Val < HostGdtr.cbGdt)
5066 {
5067 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5068 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5069 }
5070
5071 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5072 Log4(("Host SS %#08x\n", u32Val));
5073 if (u32Val < HostGdtr.cbGdt)
5074 {
5075 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5076 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5077 }
5078
5079 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5080 Log4(("Host TR %#08x\n", u32Val));
5081 if (u32Val < HostGdtr.cbGdt)
5082 {
5083 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5084 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5085 }
5086
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5088 Log4(("Host TR Base %#RHv\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5090 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5091 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5092 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5093 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5094 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5095 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5096 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5098 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5100 Log4(("Host RSP %#RHv\n", uHCReg));
5101 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5102 Log4(("Host RIP %#RHv\n", uHCReg));
5103# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5104 if (HMVMX_IS_64BIT_HOST_MODE())
5105 {
5106 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5107 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5108 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5109 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5110 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5111 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5112 }
5113# endif
5114#endif /* VBOX_STRICT */
5115 break;
5116 }
5117
5118 default:
5119 /* Impossible */
5120 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5121 break;
5122 }
5123 NOREF(pVM); NOREF(pCtx);
5124}
5125
5126
5127#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5128#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5129# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5130#endif
5131#ifdef VBOX_STRICT
5132static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5133{
5134 switch (idxField)
5135 {
5136 case VMX_VMCS_GUEST_RIP:
5137 case VMX_VMCS_GUEST_RSP:
5138 case VMX_VMCS_GUEST_SYSENTER_EIP:
5139 case VMX_VMCS_GUEST_SYSENTER_ESP:
5140 case VMX_VMCS_GUEST_GDTR_BASE:
5141 case VMX_VMCS_GUEST_IDTR_BASE:
5142 case VMX_VMCS_GUEST_CS_BASE:
5143 case VMX_VMCS_GUEST_DS_BASE:
5144 case VMX_VMCS_GUEST_ES_BASE:
5145 case VMX_VMCS_GUEST_FS_BASE:
5146 case VMX_VMCS_GUEST_GS_BASE:
5147 case VMX_VMCS_GUEST_SS_BASE:
5148 case VMX_VMCS_GUEST_LDTR_BASE:
5149 case VMX_VMCS_GUEST_TR_BASE:
5150 case VMX_VMCS_GUEST_CR3:
5151 return true;
5152 }
5153 return false;
5154}
5155
5156static bool hmR0VmxIsValidReadField(uint32_t idxField)
5157{
5158 switch (idxField)
5159 {
5160 /* Read-only fields. */
5161 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5162 return true;
5163 }
5164 /* Remaining readable fields should also be writable. */
5165 return hmR0VmxIsValidWriteField(idxField);
5166}
5167#endif /* VBOX_STRICT */
5168
5169
5170/**
5171 * Executes the specified handler in 64-bit mode.
5172 *
5173 * @returns VBox status code.
5174 * @param pVM Pointer to the VM.
5175 * @param pVCpu Pointer to the VMCPU.
5176 * @param pCtx Pointer to the guest CPU context.
5177 * @param enmOp The operation to perform.
5178 * @param cbParam Number of parameters.
5179 * @param paParam Array of 32-bit parameters.
5180 */
5181VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5182 uint32_t *paParam)
5183{
5184 int rc, rc2;
5185 PHMGLOBALCPUINFO pCpu;
5186 RTHCPHYS HCPhysCpuPage;
5187 RTCCUINTREG uOldEflags;
5188
5189 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5190 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5191 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5192 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5193
5194#ifdef VBOX_STRICT
5195 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5196 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5197
5198 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5199 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5200#endif
5201
5202 /* Disable interrupts. */
5203 uOldEflags = ASMIntDisableFlags();
5204
5205#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5206 RTCPUID idHostCpu = RTMpCpuId();
5207 CPUMR0SetLApic(pVCpu, idHostCpu);
5208#endif
5209
5210 pCpu = HMR0GetCurrentCpu();
5211 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5212
5213 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5214 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5215
5216 /* Leave VMX Root Mode. */
5217 VMXDisable();
5218
5219 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5220
5221 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5222 CPUMSetHyperEIP(pVCpu, enmOp);
5223 for (int i = (int)cbParam - 1; i >= 0; i--)
5224 CPUMPushHyper(pVCpu, paParam[i]);
5225
5226 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5227
5228 /* Call the switcher. */
5229 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5230 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5231
5232 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5233 /* Make sure the VMX instructions don't cause #UD faults. */
5234 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5235
5236 /* Re-enter VMX Root Mode */
5237 rc2 = VMXEnable(HCPhysCpuPage);
5238 if (RT_FAILURE(rc2))
5239 {
5240 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5241 ASMSetFlags(uOldEflags);
5242 return rc2;
5243 }
5244
5245 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5246 AssertRC(rc2);
5247 Assert(!(ASMGetFlags() & X86_EFL_IF));
5248 ASMSetFlags(uOldEflags);
5249 return rc;
5250}
5251
5252
5253/**
5254 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5255 * supporting 64-bit guests.
5256 *
5257 * @returns VBox status code.
5258 * @param fResume Whether to VMLAUNCH or VMRESUME.
5259 * @param pCtx Pointer to the guest-CPU context.
5260 * @param pCache Pointer to the VMCS cache.
5261 * @param pVM Pointer to the VM.
5262 * @param pVCpu Pointer to the VMCPU.
5263 */
5264DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5265{
5266 uint32_t aParam[6];
5267 PHMGLOBALCPUINFO pCpu = NULL;
5268 RTHCPHYS HCPhysCpuPage = 0;
5269 int rc = VERR_INTERNAL_ERROR_5;
5270
5271 pCpu = HMR0GetCurrentCpu();
5272 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5273
5274#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5275 pCache->uPos = 1;
5276 pCache->interPD = PGMGetInterPaeCR3(pVM);
5277 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5278#endif
5279
5280#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5281 pCache->TestIn.HCPhysCpuPage = 0;
5282 pCache->TestIn.HCPhysVmcs = 0;
5283 pCache->TestIn.pCache = 0;
5284 pCache->TestOut.HCPhysVmcs = 0;
5285 pCache->TestOut.pCache = 0;
5286 pCache->TestOut.pCtx = 0;
5287 pCache->TestOut.eflags = 0;
5288#endif
5289
5290 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5291 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5292 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5293 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5294 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5295 aParam[5] = 0;
5296
5297#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5298 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5299 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5300#endif
5301 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5302
5303#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5304 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5305 Assert(pCtx->dr[4] == 10);
5306 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5307#endif
5308
5309#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5310 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5311 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5312 pVCpu->hm.s.vmx.HCPhysVmcs));
5313 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5314 pCache->TestOut.HCPhysVmcs));
5315 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5316 pCache->TestOut.pCache));
5317 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5318 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5319 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5320 pCache->TestOut.pCtx));
5321 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5322#endif
5323 return rc;
5324}
5325
5326
5327/**
5328 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5329 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5330 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5331 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5332 *
5333 * @returns VBox status code.
5334 * @param pVM Pointer to the VM.
5335 * @param pVCpu Pointer to the VMCPU.
5336 */
5337static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5338{
5339#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5340{ \
5341 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5342 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5343 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5344 ++cReadFields; \
5345}
5346
5347 AssertPtr(pVM);
5348 AssertPtr(pVCpu);
5349 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5350 uint32_t cReadFields = 0;
5351
5352 /*
5353 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5354 * and serve to indicate exceptions to the rules.
5355 */
5356
5357 /* Guest-natural selector base fields. */
5358#if 0
5359 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5362#endif
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5375#if 0
5376 /* Unused natural width guest-state fields. */
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5379#endif
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5382
5383 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5384#if 0
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5394#endif
5395
5396 /* Natural width guest-state fields. */
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5398#if 0
5399 /* Currently unused field. */
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5401#endif
5402
5403 if (pVM->hm.s.fNestedPaging)
5404 {
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5406 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5407 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5408 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5409 }
5410 else
5411 {
5412 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5413 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5414 }
5415
5416#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5417 return VINF_SUCCESS;
5418}
5419
5420
5421/**
5422 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5423 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5424 * darwin, running 64-bit guests).
5425 *
5426 * @returns VBox status code.
5427 * @param pVCpu Pointer to the VMCPU.
5428 * @param idxField The VMCS field encoding.
5429 * @param u64Val 16, 32 or 64-bit value.
5430 */
5431VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5432{
5433 int rc;
5434 switch (idxField)
5435 {
5436 /*
5437 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5438 */
5439 /* 64-bit Control fields. */
5440 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5441 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5442 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5443 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5444 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5445 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5446 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5447 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5448 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5449 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5450 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5451 case VMX_VMCS64_CTRL_EPTP_FULL:
5452 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5453 /* 64-bit Guest-state fields. */
5454 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5455 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5456 case VMX_VMCS64_GUEST_PAT_FULL:
5457 case VMX_VMCS64_GUEST_EFER_FULL:
5458 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5459 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5460 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5461 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5462 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5463 /* 64-bit Host-state fields. */
5464 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5465 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5466 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5467 {
5468 rc = VMXWriteVmcs32(idxField, u64Val);
5469 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5470 break;
5471 }
5472
5473 /*
5474 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5475 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5476 */
5477 /* Natural-width Guest-state fields. */
5478 case VMX_VMCS_GUEST_CR3:
5479 case VMX_VMCS_GUEST_ES_BASE:
5480 case VMX_VMCS_GUEST_CS_BASE:
5481 case VMX_VMCS_GUEST_SS_BASE:
5482 case VMX_VMCS_GUEST_DS_BASE:
5483 case VMX_VMCS_GUEST_FS_BASE:
5484 case VMX_VMCS_GUEST_GS_BASE:
5485 case VMX_VMCS_GUEST_LDTR_BASE:
5486 case VMX_VMCS_GUEST_TR_BASE:
5487 case VMX_VMCS_GUEST_GDTR_BASE:
5488 case VMX_VMCS_GUEST_IDTR_BASE:
5489 case VMX_VMCS_GUEST_RSP:
5490 case VMX_VMCS_GUEST_RIP:
5491 case VMX_VMCS_GUEST_SYSENTER_ESP:
5492 case VMX_VMCS_GUEST_SYSENTER_EIP:
5493 {
5494 if (!(u64Val >> 32))
5495 {
5496 /* If this field is 64-bit, VT-x will zero out the top bits. */
5497 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5498 }
5499 else
5500 {
5501 /* Assert that only the 32->64 switcher case should ever come here. */
5502 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5503 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5504 }
5505 break;
5506 }
5507
5508 default:
5509 {
5510 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5511 rc = VERR_INVALID_PARAMETER;
5512 break;
5513 }
5514 }
5515 AssertRCReturn(rc, rc);
5516 return rc;
5517}
5518
5519
5520/**
5521 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5522 * hosts (except darwin) for 64-bit guests.
5523 *
5524 * @param pVCpu Pointer to the VMCPU.
5525 * @param idxField The VMCS field encoding.
5526 * @param u64Val 16, 32 or 64-bit value.
5527 */
5528VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5529{
5530 AssertPtr(pVCpu);
5531 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5532
5533 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5534 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5535
5536 /* Make sure there are no duplicates. */
5537 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5538 {
5539 if (pCache->Write.aField[i] == idxField)
5540 {
5541 pCache->Write.aFieldVal[i] = u64Val;
5542 return VINF_SUCCESS;
5543 }
5544 }
5545
5546 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5547 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5548 pCache->Write.cValidEntries++;
5549 return VINF_SUCCESS;
5550}
5551
5552/* Enable later when the assembly code uses these as callbacks. */
5553#if 0
5554/*
5555 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5556 *
5557 * @param pVCpu Pointer to the VMCPU.
5558 * @param pCache Pointer to the VMCS cache.
5559 *
5560 * @remarks No-long-jump zone!!!
5561 */
5562VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5563{
5564 AssertPtr(pCache);
5565 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5566 {
5567 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5568 AssertRC(rc);
5569 }
5570 pCache->Write.cValidEntries = 0;
5571}
5572
5573
5574/**
5575 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5576 *
5577 * @param pVCpu Pointer to the VMCPU.
5578 * @param pCache Pointer to the VMCS cache.
5579 *
5580 * @remarks No-long-jump zone!!!
5581 */
5582VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5583{
5584 AssertPtr(pCache);
5585 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5586 {
5587 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5588 AssertRC(rc);
5589 }
5590}
5591#endif
5592#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5593
5594
5595/**
5596 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5597 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5598 * timer.
5599 *
5600 * @returns VBox status code.
5601 * @param pVCpu Pointer to the VMCPU.
5602 *
5603 * @remarks No-long-jump zone!!!
5604 */
5605static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5606{
5607 int rc = VERR_INTERNAL_ERROR_5;
5608 bool fOffsettedTsc = false;
5609 bool fParavirtTsc = false;
5610 PVM pVM = pVCpu->CTX_SUFF(pVM);
5611 if (pVM->hm.s.vmx.fUsePreemptTimer)
5612 {
5613 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5614 &pVCpu->hm.s.vmx.u64TSCOffset);
5615
5616 /* Make sure the returned values have sane upper and lower boundaries. */
5617 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5618 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5619 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5620 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5621
5622 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5623 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5624 }
5625 else
5626 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5627
5628#if 1
5629 if (fParavirtTsc)
5630 {
5631#if 1
5632 uint64_t const u64CurTsc = ASMReadTSC();
5633 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5634 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5635 {
5636 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5637 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5638 }
5639
5640 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5641#endif
5642 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5643 AssertRC(rc);
5644 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5645 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5646
5647 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5648 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5649 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5650 }
5651 else
5652#else
5653 if (fParavirtTsc)
5654 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5655#endif
5656 if (fOffsettedTsc)
5657 {
5658 uint64_t u64CurTSC = ASMReadTSC();
5659 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5660 {
5661 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5662 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5663
5664 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5665 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5666 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5667 }
5668 else
5669 {
5670 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5671 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5672 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5673 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5674 }
5675 }
5676 else
5677 {
5678 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5679 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5680 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5681 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5682 }
5683}
5684
5685
5686/**
5687 * Determines if an exception is a contributory exception. Contributory
5688 * exceptions are ones which can cause double-faults. Page-fault is
5689 * intentionally not included here as it's a conditional contributory exception.
5690 *
5691 * @returns true if the exception is contributory, false otherwise.
5692 * @param uVector The exception vector.
5693 */
5694DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5695{
5696 switch (uVector)
5697 {
5698 case X86_XCPT_GP:
5699 case X86_XCPT_SS:
5700 case X86_XCPT_NP:
5701 case X86_XCPT_TS:
5702 case X86_XCPT_DE:
5703 return true;
5704 default:
5705 break;
5706 }
5707 return false;
5708}
5709
5710
5711/**
5712 * Sets an event as a pending event to be injected into the guest.
5713 *
5714 * @param pVCpu Pointer to the VMCPU.
5715 * @param u32IntInfo The VM-entry interruption-information field.
5716 * @param cbInstr The VM-entry instruction length in bytes (for software
5717 * interrupts, exceptions and privileged software
5718 * exceptions).
5719 * @param u32ErrCode The VM-entry exception error code.
5720 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5721 * page-fault.
5722 *
5723 * @remarks Statistics counter assumes this is a guest event being injected or
5724 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5725 * always incremented.
5726 */
5727DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5728 RTGCUINTPTR GCPtrFaultAddress)
5729{
5730 Assert(!pVCpu->hm.s.Event.fPending);
5731 pVCpu->hm.s.Event.fPending = true;
5732 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5733 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5734 pVCpu->hm.s.Event.cbInstr = cbInstr;
5735 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5736
5737 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5738}
5739
5740
5741/**
5742 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5743 *
5744 * @param pVCpu Pointer to the VMCPU.
5745 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5746 * out-of-sync. Make sure to update the required fields
5747 * before using them.
5748 */
5749DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5750{
5751 NOREF(pMixedCtx);
5752 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5753 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5754 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5755 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5756}
5757
5758
5759/**
5760 * Handle a condition that occurred while delivering an event through the guest
5761 * IDT.
5762 *
5763 * @returns VBox status code (informational error codes included).
5764 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5765 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5766 * continue execution of the guest which will delivery the #DF.
5767 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5768 *
5769 * @param pVCpu Pointer to the VMCPU.
5770 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5771 * out-of-sync. Make sure to update the required fields
5772 * before using them.
5773 * @param pVmxTransient Pointer to the VMX transient structure.
5774 *
5775 * @remarks No-long-jump zone!!!
5776 */
5777static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5778{
5779 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5780
5781 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5782 AssertRCReturn(rc, rc);
5783 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5784 {
5785 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5786 AssertRCReturn(rc, rc);
5787
5788 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5789 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5790
5791 typedef enum
5792 {
5793 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5794 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5795 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5796 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5797 } VMXREFLECTXCPT;
5798
5799 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5800 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5801 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5802 {
5803 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5804 {
5805 enmReflect = VMXREFLECTXCPT_XCPT;
5806#ifdef VBOX_STRICT
5807 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5808 && uExitVector == X86_XCPT_PF)
5809 {
5810 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5811 }
5812#endif
5813 if ( uExitVector == X86_XCPT_PF
5814 && uIdtVector == X86_XCPT_PF)
5815 {
5816 pVmxTransient->fVectoringPF = true;
5817 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5818 }
5819 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5820 && hmR0VmxIsContributoryXcpt(uExitVector)
5821 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5822 || uIdtVector == X86_XCPT_PF))
5823 {
5824 enmReflect = VMXREFLECTXCPT_DF;
5825 }
5826 else if (uIdtVector == X86_XCPT_DF)
5827 enmReflect = VMXREFLECTXCPT_TF;
5828 }
5829 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5830 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5831 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5832 {
5833 /*
5834 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5835 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5836 */
5837 enmReflect = VMXREFLECTXCPT_XCPT;
5838 }
5839 }
5840 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5841 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5842 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5843 {
5844 /*
5845 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5846 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5847 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5848 */
5849 enmReflect = VMXREFLECTXCPT_XCPT;
5850 }
5851
5852 /*
5853 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5854 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5855 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5856 *
5857 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5858 */
5859 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5860 && enmReflect == VMXREFLECTXCPT_XCPT
5861 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5862 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5863 {
5864 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5865 }
5866
5867 switch (enmReflect)
5868 {
5869 case VMXREFLECTXCPT_XCPT:
5870 {
5871 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5872 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5873 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5874
5875 uint32_t u32ErrCode = 0;
5876 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5877 {
5878 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5879 AssertRCReturn(rc, rc);
5880 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5881 }
5882
5883 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5884 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5885 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5886 rc = VINF_SUCCESS;
5887 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5888 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5889
5890 break;
5891 }
5892
5893 case VMXREFLECTXCPT_DF:
5894 {
5895 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5896 rc = VINF_HM_DOUBLE_FAULT;
5897 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5898 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5899
5900 break;
5901 }
5902
5903 case VMXREFLECTXCPT_TF:
5904 {
5905 rc = VINF_EM_RESET;
5906 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5907 uExitVector));
5908 break;
5909 }
5910
5911 default:
5912 Assert(rc == VINF_SUCCESS);
5913 break;
5914 }
5915 }
5916 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5917 && uExitVector != X86_XCPT_DF
5918 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5919 {
5920 /*
5921 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5922 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5923 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5924 */
5925 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5926 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5927 }
5928
5929 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5930 return rc;
5931}
5932
5933
5934/**
5935 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5936 *
5937 * @returns VBox status code.
5938 * @param pVCpu Pointer to the VMCPU.
5939 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5940 * out-of-sync. Make sure to update the required fields
5941 * before using them.
5942 *
5943 * @remarks No-long-jump zone!!!
5944 */
5945static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5946{
5947 NOREF(pMixedCtx);
5948
5949 /*
5950 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5951 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5952 */
5953 VMMRZCallRing3Disable(pVCpu);
5954 HM_DISABLE_PREEMPT_IF_NEEDED();
5955
5956 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5957 {
5958 uint32_t uVal = 0;
5959 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5960 AssertRCReturn(rc, rc);
5961
5962 uint32_t uShadow = 0;
5963 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5964 AssertRCReturn(rc, rc);
5965
5966 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5967 CPUMSetGuestCR0(pVCpu, uVal);
5968 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5969 }
5970
5971 HM_RESTORE_PREEMPT_IF_NEEDED();
5972 VMMRZCallRing3Enable(pVCpu);
5973 return VINF_SUCCESS;
5974}
5975
5976
5977/**
5978 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5979 *
5980 * @returns VBox status code.
5981 * @param pVCpu Pointer to the VMCPU.
5982 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5983 * out-of-sync. Make sure to update the required fields
5984 * before using them.
5985 *
5986 * @remarks No-long-jump zone!!!
5987 */
5988static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5989{
5990 NOREF(pMixedCtx);
5991
5992 int rc = VINF_SUCCESS;
5993 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5994 {
5995 uint32_t uVal = 0;
5996 uint32_t uShadow = 0;
5997 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5998 AssertRCReturn(rc, rc);
5999 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6000 AssertRCReturn(rc, rc);
6001
6002 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6003 CPUMSetGuestCR4(pVCpu, uVal);
6004 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6005 }
6006 return rc;
6007}
6008
6009
6010/**
6011 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6012 *
6013 * @returns VBox status code.
6014 * @param pVCpu Pointer to the VMCPU.
6015 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6016 * out-of-sync. Make sure to update the required fields
6017 * before using them.
6018 *
6019 * @remarks No-long-jump zone!!!
6020 */
6021static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6022{
6023 int rc = VINF_SUCCESS;
6024 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6025 {
6026 uint64_t u64Val = 0;
6027 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6028 AssertRCReturn(rc, rc);
6029
6030 pMixedCtx->rip = u64Val;
6031 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6032 }
6033 return rc;
6034}
6035
6036
6037/**
6038 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6039 *
6040 * @returns VBox status code.
6041 * @param pVCpu Pointer to the VMCPU.
6042 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6043 * out-of-sync. Make sure to update the required fields
6044 * before using them.
6045 *
6046 * @remarks No-long-jump zone!!!
6047 */
6048static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6049{
6050 int rc = VINF_SUCCESS;
6051 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6052 {
6053 uint64_t u64Val = 0;
6054 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6055 AssertRCReturn(rc, rc);
6056
6057 pMixedCtx->rsp = u64Val;
6058 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6059 }
6060 return rc;
6061}
6062
6063
6064/**
6065 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6066 *
6067 * @returns VBox status code.
6068 * @param pVCpu Pointer to the VMCPU.
6069 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6070 * out-of-sync. Make sure to update the required fields
6071 * before using them.
6072 *
6073 * @remarks No-long-jump zone!!!
6074 */
6075static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6076{
6077 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6078 {
6079 uint32_t uVal = 0;
6080 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6081 AssertRCReturn(rc, rc);
6082
6083 pMixedCtx->eflags.u32 = uVal;
6084 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6085 {
6086 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6087 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6088
6089 pMixedCtx->eflags.Bits.u1VM = 0;
6090 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6091 }
6092
6093 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6094 }
6095 return VINF_SUCCESS;
6096}
6097
6098
6099/**
6100 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6101 * guest-CPU context.
6102 */
6103DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6104{
6105 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6106 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6107 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6108 return rc;
6109}
6110
6111
6112/**
6113 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6114 * from the guest-state area in the VMCS.
6115 *
6116 * @param pVCpu Pointer to the VMCPU.
6117 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6118 * out-of-sync. Make sure to update the required fields
6119 * before using them.
6120 *
6121 * @remarks No-long-jump zone!!!
6122 */
6123static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6124{
6125 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6126 {
6127 uint32_t uIntrState = 0;
6128 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6129 AssertRC(rc);
6130
6131 if (!uIntrState)
6132 {
6133 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6134 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6135
6136 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6137 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6138 }
6139 else
6140 {
6141 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6142 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6143 {
6144 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6145 AssertRC(rc);
6146 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6147 AssertRC(rc);
6148
6149 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6150 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6151 }
6152
6153 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6154 {
6155 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6156 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6157 }
6158 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6159 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6160 }
6161
6162 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6163 }
6164}
6165
6166
6167/**
6168 * Saves the guest's activity state.
6169 *
6170 * @returns VBox status code.
6171 * @param pVCpu Pointer to the VMCPU.
6172 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6173 * out-of-sync. Make sure to update the required fields
6174 * before using them.
6175 *
6176 * @remarks No-long-jump zone!!!
6177 */
6178static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6179{
6180 NOREF(pMixedCtx);
6181 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6182 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6183 return VINF_SUCCESS;
6184}
6185
6186
6187/**
6188 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6189 * the current VMCS into the guest-CPU context.
6190 *
6191 * @returns VBox status code.
6192 * @param pVCpu Pointer to the VMCPU.
6193 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6194 * out-of-sync. Make sure to update the required fields
6195 * before using them.
6196 *
6197 * @remarks No-long-jump zone!!!
6198 */
6199static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6200{
6201 int rc = VINF_SUCCESS;
6202 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6203 {
6204 uint32_t u32Val = 0;
6205 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6206 pMixedCtx->SysEnter.cs = u32Val;
6207 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6208 }
6209
6210 uint64_t u64Val = 0;
6211 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6212 {
6213 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6214 pMixedCtx->SysEnter.eip = u64Val;
6215 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6216 }
6217 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6218 {
6219 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6220 pMixedCtx->SysEnter.esp = u64Val;
6221 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6222 }
6223 return rc;
6224}
6225
6226
6227/**
6228 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6229 * the CPU back into the guest-CPU context.
6230 *
6231 * @returns VBox status code.
6232 * @param pVCpu Pointer to the VMCPU.
6233 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6234 * out-of-sync. Make sure to update the required fields
6235 * before using them.
6236 *
6237 * @remarks No-long-jump zone!!!
6238 */
6239static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6240{
6241#if HC_ARCH_BITS == 64
6242 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6243 {
6244 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6245 VMMRZCallRing3Disable(pVCpu);
6246 HM_DISABLE_PREEMPT_IF_NEEDED();
6247
6248 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6249 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6250 {
6251 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6252 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6253 }
6254
6255 HM_RESTORE_PREEMPT_IF_NEEDED();
6256 VMMRZCallRing3Enable(pVCpu);
6257 }
6258 else
6259 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6260#else
6261 NOREF(pMixedCtx);
6262 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6263#endif
6264
6265 return VINF_SUCCESS;
6266}
6267
6268
6269/**
6270 * Saves the auto load/store'd guest MSRs from the current VMCS into
6271 * the guest-CPU context.
6272 *
6273 * @returns VBox status code.
6274 * @param pVCpu Pointer to the VMCPU.
6275 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6276 * out-of-sync. Make sure to update the required fields
6277 * before using them.
6278 *
6279 * @remarks No-long-jump zone!!!
6280 */
6281static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282{
6283 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6284 return VINF_SUCCESS;
6285
6286 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6287 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6288 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6289 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6290 {
6291 switch (pMsr->u32Msr)
6292 {
6293 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6294 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6295 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6296 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6297 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6298 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6299 break;
6300
6301 default:
6302 {
6303 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6304 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6305 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6306 }
6307 }
6308 }
6309
6310 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6311 return VINF_SUCCESS;
6312}
6313
6314
6315/**
6316 * Saves the guest control registers from the current VMCS into the guest-CPU
6317 * context.
6318 *
6319 * @returns VBox status code.
6320 * @param pVCpu Pointer to the VMCPU.
6321 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6322 * out-of-sync. Make sure to update the required fields
6323 * before using them.
6324 *
6325 * @remarks No-long-jump zone!!!
6326 */
6327static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6328{
6329 /* Guest CR0. Guest FPU. */
6330 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6331 AssertRCReturn(rc, rc);
6332
6333 /* Guest CR4. */
6334 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6335 AssertRCReturn(rc, rc);
6336
6337 /* Guest CR2 - updated always during the world-switch or in #PF. */
6338 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6339 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6340 {
6341 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6342 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6343
6344 PVM pVM = pVCpu->CTX_SUFF(pVM);
6345 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6346 || ( pVM->hm.s.fNestedPaging
6347 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6348 {
6349 uint64_t u64Val = 0;
6350 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6351 if (pMixedCtx->cr3 != u64Val)
6352 {
6353 CPUMSetGuestCR3(pVCpu, u64Val);
6354 if (VMMRZCallRing3IsEnabled(pVCpu))
6355 {
6356 PGMUpdateCR3(pVCpu, u64Val);
6357 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6358 }
6359 else
6360 {
6361 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6362 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6363 }
6364 }
6365
6366 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6367 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6368 {
6369 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6370 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6371 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6372 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6373
6374 if (VMMRZCallRing3IsEnabled(pVCpu))
6375 {
6376 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6377 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6378 }
6379 else
6380 {
6381 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6382 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6383 }
6384 }
6385 }
6386
6387 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6388 }
6389
6390 /*
6391 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6392 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6393 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6394 *
6395 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6396 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6397 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6398 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6399 *
6400 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6401 */
6402 if (VMMRZCallRing3IsEnabled(pVCpu))
6403 {
6404 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6405 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6406
6407 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6408 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6409
6410 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6411 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6412 }
6413
6414 return rc;
6415}
6416
6417
6418/**
6419 * Reads a guest segment register from the current VMCS into the guest-CPU
6420 * context.
6421 *
6422 * @returns VBox status code.
6423 * @param pVCpu Pointer to the VMCPU.
6424 * @param idxSel Index of the selector in the VMCS.
6425 * @param idxLimit Index of the segment limit in the VMCS.
6426 * @param idxBase Index of the segment base in the VMCS.
6427 * @param idxAccess Index of the access rights of the segment in the VMCS.
6428 * @param pSelReg Pointer to the segment selector.
6429 *
6430 * @remarks No-long-jump zone!!!
6431 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6432 * macro as that takes care of whether to read from the VMCS cache or
6433 * not.
6434 */
6435DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6436 PCPUMSELREG pSelReg)
6437{
6438 NOREF(pVCpu);
6439
6440 uint32_t u32Val = 0;
6441 int rc = VMXReadVmcs32(idxSel, &u32Val);
6442 AssertRCReturn(rc, rc);
6443 pSelReg->Sel = (uint16_t)u32Val;
6444 pSelReg->ValidSel = (uint16_t)u32Val;
6445 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6446
6447 rc = VMXReadVmcs32(idxLimit, &u32Val);
6448 AssertRCReturn(rc, rc);
6449 pSelReg->u32Limit = u32Val;
6450
6451 uint64_t u64Val = 0;
6452 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6453 AssertRCReturn(rc, rc);
6454 pSelReg->u64Base = u64Val;
6455
6456 rc = VMXReadVmcs32(idxAccess, &u32Val);
6457 AssertRCReturn(rc, rc);
6458 pSelReg->Attr.u = u32Val;
6459
6460 /*
6461 * If VT-x marks the segment as unusable, most other bits remain undefined:
6462 * - For CS the L, D and G bits have meaning.
6463 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6464 * - For the remaining data segments no bits are defined.
6465 *
6466 * The present bit and the unusable bit has been observed to be set at the
6467 * same time (the selector was supposed to be invalid as we started executing
6468 * a V8086 interrupt in ring-0).
6469 *
6470 * What should be important for the rest of the VBox code, is that the P bit is
6471 * cleared. Some of the other VBox code recognizes the unusable bit, but
6472 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6473 * safe side here, we'll strip off P and other bits we don't care about. If
6474 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6475 *
6476 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6477 */
6478 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6479 {
6480 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6481
6482 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6483 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6484 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6485
6486 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6487#ifdef DEBUG_bird
6488 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6489 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6490 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6491#endif
6492 }
6493 return VINF_SUCCESS;
6494}
6495
6496
6497#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6498# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6499 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6500 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6501#else
6502# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6503 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6504 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6505#endif
6506
6507
6508/**
6509 * Saves the guest segment registers from the current VMCS into the guest-CPU
6510 * context.
6511 *
6512 * @returns VBox status code.
6513 * @param pVCpu Pointer to the VMCPU.
6514 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6515 * out-of-sync. Make sure to update the required fields
6516 * before using them.
6517 *
6518 * @remarks No-long-jump zone!!!
6519 */
6520static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6521{
6522 /* Guest segment registers. */
6523 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6524 {
6525 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6526 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6527 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6528 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6529 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6530 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6531 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6532
6533 /* Restore segment attributes for real-on-v86 mode hack. */
6534 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6535 {
6536 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6537 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6538 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6539 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6540 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6541 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6542 }
6543 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6544 }
6545
6546 return VINF_SUCCESS;
6547}
6548
6549
6550/**
6551 * Saves the guest descriptor table registers and task register from the current
6552 * VMCS into the guest-CPU context.
6553 *
6554 * @returns VBox status code.
6555 * @param pVCpu Pointer to the VMCPU.
6556 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6557 * out-of-sync. Make sure to update the required fields
6558 * before using them.
6559 *
6560 * @remarks No-long-jump zone!!!
6561 */
6562static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6563{
6564 int rc = VINF_SUCCESS;
6565
6566 /* Guest LDTR. */
6567 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6568 {
6569 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6570 AssertRCReturn(rc, rc);
6571 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6572 }
6573
6574 /* Guest GDTR. */
6575 uint64_t u64Val = 0;
6576 uint32_t u32Val = 0;
6577 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6578 {
6579 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6580 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6581 pMixedCtx->gdtr.pGdt = u64Val;
6582 pMixedCtx->gdtr.cbGdt = u32Val;
6583 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6584 }
6585
6586 /* Guest IDTR. */
6587 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6588 {
6589 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6590 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6591 pMixedCtx->idtr.pIdt = u64Val;
6592 pMixedCtx->idtr.cbIdt = u32Val;
6593 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6594 }
6595
6596 /* Guest TR. */
6597 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6598 {
6599 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6600 AssertRCReturn(rc, rc);
6601
6602 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6603 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6604 {
6605 rc = VMXLOCAL_READ_SEG(TR, tr);
6606 AssertRCReturn(rc, rc);
6607 }
6608 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6609 }
6610 return rc;
6611}
6612
6613#undef VMXLOCAL_READ_SEG
6614
6615
6616/**
6617 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6618 * context.
6619 *
6620 * @returns VBox status code.
6621 * @param pVCpu Pointer to the VMCPU.
6622 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6623 * out-of-sync. Make sure to update the required fields
6624 * before using them.
6625 *
6626 * @remarks No-long-jump zone!!!
6627 */
6628static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6629{
6630 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6631 {
6632 if (!pVCpu->hm.s.fUsingHyperDR7)
6633 {
6634 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6635 uint32_t u32Val;
6636 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6637 pMixedCtx->dr[7] = u32Val;
6638 }
6639
6640 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6641 }
6642 return VINF_SUCCESS;
6643}
6644
6645
6646/**
6647 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6648 *
6649 * @returns VBox status code.
6650 * @param pVCpu Pointer to the VMCPU.
6651 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6652 * out-of-sync. Make sure to update the required fields
6653 * before using them.
6654 *
6655 * @remarks No-long-jump zone!!!
6656 */
6657static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6658{
6659 NOREF(pMixedCtx);
6660
6661 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6662 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6663 return VINF_SUCCESS;
6664}
6665
6666
6667/**
6668 * Saves the entire guest state from the currently active VMCS into the
6669 * guest-CPU context. This essentially VMREADs all guest-data.
6670 *
6671 * @returns VBox status code.
6672 * @param pVCpu Pointer to the VMCPU.
6673 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6674 * out-of-sync. Make sure to update the required fields
6675 * before using them.
6676 */
6677static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6678{
6679 Assert(pVCpu);
6680 Assert(pMixedCtx);
6681
6682 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6683 return VINF_SUCCESS;
6684
6685 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6686 again on the ring-3 callback path, there is no real need to. */
6687 if (VMMRZCallRing3IsEnabled(pVCpu))
6688 VMMR0LogFlushDisable(pVCpu);
6689 else
6690 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6691 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6692
6693 /* In case we get preempted before saving the interruptibility-state, do it here
6694 otherwise we lose the info. in the VMCS if we get rescheduled on a different host CPU. */
6695 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6696
6697 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6698 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6699
6700 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6701 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6702
6703 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6704 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6705
6706 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6707 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6708
6709 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6710 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6711
6712 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6713 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6714
6715 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6716 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6717
6718 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6719 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6720
6721 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6722 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6723
6724 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6725 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6726
6727 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6728 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6729
6730 if (VMMRZCallRing3IsEnabled(pVCpu))
6731 VMMR0LogFlushEnable(pVCpu);
6732
6733 return rc;
6734}
6735
6736
6737/**
6738 * Check per-VM and per-VCPU force flag actions that require us to go back to
6739 * ring-3 for one reason or another.
6740 *
6741 * @returns VBox status code (information status code included).
6742 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6743 * ring-3.
6744 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6745 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6746 * interrupts)
6747 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6748 * all EMTs to be in ring-3.
6749 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6750 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6751 * to the EM loop.
6752 *
6753 * @param pVM Pointer to the VM.
6754 * @param pVCpu Pointer to the VMCPU.
6755 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6756 * out-of-sync. Make sure to update the required fields
6757 * before using them.
6758 */
6759static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6760{
6761 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6762
6763 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6764 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6765 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6766 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6767 {
6768 /* We need the control registers now, make sure the guest-CPU context is updated. */
6769 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6770 AssertRCReturn(rc3, rc3);
6771
6772 /* Pending HM CR3 sync. */
6773 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6774 {
6775 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6776 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6777 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6778 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6779 }
6780
6781 /* Pending HM PAE PDPEs. */
6782 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6783 {
6784 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6785 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6786 }
6787
6788 /* Pending PGM C3 sync. */
6789 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6790 {
6791 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6792 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6793 if (rc2 != VINF_SUCCESS)
6794 {
6795 AssertRC(rc2);
6796 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6797 return rc2;
6798 }
6799 }
6800
6801 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6802 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6803 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6804 {
6805 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6806 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6807 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6808 return rc2;
6809 }
6810
6811 /* Pending VM request packets, such as hardware interrupts. */
6812 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6813 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6814 {
6815 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6816 return VINF_EM_PENDING_REQUEST;
6817 }
6818
6819 /* Pending PGM pool flushes. */
6820 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6821 {
6822 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6823 return VINF_PGM_POOL_FLUSH_PENDING;
6824 }
6825
6826 /* Pending DMA requests. */
6827 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6828 {
6829 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6830 return VINF_EM_RAW_TO_R3;
6831 }
6832 }
6833
6834 return VINF_SUCCESS;
6835}
6836
6837
6838/**
6839 * Converts any TRPM trap into a pending HM event. This is typically used when
6840 * entering from ring-3 (not longjmp returns).
6841 *
6842 * @param pVCpu Pointer to the VMCPU.
6843 */
6844static int hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6845{
6846 Assert(TRPMHasTrap(pVCpu));
6847 Assert(!pVCpu->hm.s.Event.fPending);
6848
6849 uint8_t uVector;
6850 TRPMEVENT enmTrpmEvent;
6851 RTGCUINT uErrCode;
6852 RTGCUINTPTR GCPtrFaultAddress;
6853 uint8_t cbInstr;
6854
6855 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6856 AssertRC(rc);
6857
6858 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6859 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6860 if (enmTrpmEvent == TRPM_TRAP)
6861 {
6862 switch (uVector)
6863 {
6864 case X86_XCPT_NMI:
6865 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6866 break;
6867
6868 case X86_XCPT_BP:
6869 case X86_XCPT_OF:
6870 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6871 break;
6872
6873 case X86_XCPT_PF:
6874 case X86_XCPT_DF:
6875 case X86_XCPT_TS:
6876 case X86_XCPT_NP:
6877 case X86_XCPT_SS:
6878 case X86_XCPT_GP:
6879 case X86_XCPT_AC:
6880 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6881 /* no break! */
6882 default:
6883 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6884 break;
6885 }
6886 }
6887 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6888 {
6889 /** @todo r=ramshankar: Make this a strict-build assert after this bug is
6890 * fixed. See @bugref{7317}. */
6891 uint32_t uEFlags = CPUMGetGuestEFlags(pVCpu);
6892 if (!(uEFlags & X86_EFL_IF))
6893 {
6894 Log4(("hmR0VmxTrpmTrapToPendingEvent: TRPM injecting an external interrupt when interrupts are disabled!?!"));
6895 return VERR_VMX_IPE_5;
6896 }
6897
6898 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6899 }
6900 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6901 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6902 else
6903 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6904
6905 rc = TRPMResetTrap(pVCpu);
6906 AssertRC(rc);
6907 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6908 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6909
6910 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6911 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6912 return VINF_SUCCESS;
6913}
6914
6915
6916/**
6917 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6918 * VT-x to execute any instruction.
6919 *
6920 * @param pvCpu Pointer to the VMCPU.
6921 */
6922static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6923{
6924 Assert(pVCpu->hm.s.Event.fPending);
6925
6926 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6927 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6928 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6929 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6930
6931 /* If a trap was already pending, we did something wrong! */
6932 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6933
6934 TRPMEVENT enmTrapType;
6935 switch (uVectorType)
6936 {
6937 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6938 enmTrapType = TRPM_HARDWARE_INT;
6939 break;
6940
6941 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6942 enmTrapType = TRPM_SOFTWARE_INT;
6943 break;
6944
6945 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6946 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6947 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6948 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6949 enmTrapType = TRPM_TRAP;
6950 break;
6951
6952 default:
6953 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6954 enmTrapType = TRPM_32BIT_HACK;
6955 break;
6956 }
6957
6958 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6959
6960 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6961 AssertRC(rc);
6962
6963 if (fErrorCodeValid)
6964 TRPMSetErrorCode(pVCpu, uErrorCode);
6965
6966 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6967 && uVector == X86_XCPT_PF)
6968 {
6969 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6970 }
6971 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6972 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6973 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6974 {
6975 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6976 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6977 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6978 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6979 }
6980 pVCpu->hm.s.Event.fPending = false;
6981}
6982
6983
6984/**
6985 * Does the necessary state syncing before returning to ring-3 for any reason
6986 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6987 *
6988 * @returns VBox status code.
6989 * @param pVM Pointer to the VM.
6990 * @param pVCpu Pointer to the VMCPU.
6991 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6992 * be out-of-sync. Make sure to update the required
6993 * fields before using them.
6994 * @param fSaveGuestState Whether to save the guest state or not.
6995 *
6996 * @remarks No-long-jmp zone!!!
6997 */
6998static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6999{
7000 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7001 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7002
7003 RTCPUID idCpu = RTMpCpuId();
7004 Log4Func(("HostCpuId=%u\n", idCpu));
7005
7006 /*
7007 * !!! IMPORTANT !!!
7008 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7009 */
7010
7011 /* Save the guest state if necessary. */
7012 if ( fSaveGuestState
7013 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7014 {
7015 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7016 AssertRCReturn(rc, rc);
7017 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7018 }
7019
7020 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7021 if (CPUMIsGuestFPUStateActive(pVCpu))
7022 {
7023 /* We shouldn't reload CR0 without saving it first. */
7024 if (!fSaveGuestState)
7025 {
7026 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7027 AssertRCReturn(rc, rc);
7028 }
7029 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7030 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7031 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7032 }
7033
7034 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7035#ifdef VBOX_STRICT
7036 if (CPUMIsHyperDebugStateActive(pVCpu))
7037 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7038#endif
7039 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7040 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7041 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7042 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7043
7044#if HC_ARCH_BITS == 64
7045 /* Restore host-state bits that VT-x only restores partially. */
7046 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7047 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7048 {
7049 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7050 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7051 }
7052 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7053#endif
7054
7055#if HC_ARCH_BITS == 64
7056 /* Restore the host MSRs as we're leaving VT-x context. */
7057 if ( pVM->hm.s.fAllow64BitGuests
7058 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7059 {
7060 /* We shouldn't reload the guest MSRs without saving it first. */
7061 if (!fSaveGuestState)
7062 {
7063 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7064 AssertRCReturn(rc, rc);
7065 }
7066 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7067 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7068 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
7069 }
7070#endif
7071
7072 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7073 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7074
7075 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7076 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7077 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7078 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7079 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7080 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7081 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7082 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7083
7084 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7085
7086 /** @todo This partially defeats the purpose of having preemption hooks.
7087 * The problem is, deregistering the hooks should be moved to a place that
7088 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7089 * context.
7090 */
7091 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7092 {
7093 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7094 AssertRCReturn(rc, rc);
7095
7096 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7097 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7098 }
7099 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7100 NOREF(idCpu);
7101
7102 return VINF_SUCCESS;
7103}
7104
7105
7106/**
7107 * Leaves the VT-x session.
7108 *
7109 * @returns VBox status code.
7110 * @param pVM Pointer to the VM.
7111 * @param pVCpu Pointer to the VMCPU.
7112 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7113 * out-of-sync. Make sure to update the required fields
7114 * before using them.
7115 *
7116 * @remarks No-long-jmp zone!!!
7117 */
7118DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7119{
7120 HM_DISABLE_PREEMPT_IF_NEEDED();
7121 HMVMX_ASSERT_CPU_SAFE();
7122 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7123 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7124
7125 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7126 and done this from the VMXR0ThreadCtxCallback(). */
7127 if (!pVCpu->hm.s.fLeaveDone)
7128 {
7129 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7130 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7131 pVCpu->hm.s.fLeaveDone = true;
7132 }
7133 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7134
7135 /*
7136 * !!! IMPORTANT !!!
7137 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7138 */
7139
7140 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7141 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7142 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7143 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7144 VMMR0ThreadCtxHooksDeregister(pVCpu);
7145
7146 /* Leave HM context. This takes care of local init (term). */
7147 int rc = HMR0LeaveCpu(pVCpu);
7148
7149 HM_RESTORE_PREEMPT_IF_NEEDED();
7150
7151 return rc;
7152}
7153
7154
7155/**
7156 * Does the necessary state syncing before doing a longjmp to ring-3.
7157 *
7158 * @returns VBox status code.
7159 * @param pVM Pointer to the VM.
7160 * @param pVCpu Pointer to the VMCPU.
7161 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7162 * out-of-sync. Make sure to update the required fields
7163 * before using them.
7164 *
7165 * @remarks No-long-jmp zone!!!
7166 */
7167DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7168{
7169 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7170}
7171
7172
7173/**
7174 * Take necessary actions before going back to ring-3.
7175 *
7176 * An action requires us to go back to ring-3. This function does the necessary
7177 * steps before we can safely return to ring-3. This is not the same as longjmps
7178 * to ring-3, this is voluntary and prepares the guest so it may continue
7179 * executing outside HM (recompiler/IEM).
7180 *
7181 * @returns VBox status code.
7182 * @param pVM Pointer to the VM.
7183 * @param pVCpu Pointer to the VMCPU.
7184 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7185 * out-of-sync. Make sure to update the required fields
7186 * before using them.
7187 * @param rcExit The reason for exiting to ring-3. Can be
7188 * VINF_VMM_UNKNOWN_RING3_CALL.
7189 */
7190static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7191{
7192 Assert(pVM);
7193 Assert(pVCpu);
7194 Assert(pMixedCtx);
7195 HMVMX_ASSERT_PREEMPT_SAFE();
7196
7197 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7198 {
7199 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7200 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7201 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7202 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7203 }
7204
7205 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7206 VMMRZCallRing3Disable(pVCpu);
7207 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7208
7209 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7210 if (pVCpu->hm.s.Event.fPending)
7211 {
7212 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7213 Assert(!pVCpu->hm.s.Event.fPending);
7214 }
7215
7216 /* Save guest state and restore host state bits. */
7217 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7218 AssertRCReturn(rc, rc);
7219 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7220
7221 /* Sync recompiler state. */
7222 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7223 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7224 | CPUM_CHANGED_LDTR
7225 | CPUM_CHANGED_GDTR
7226 | CPUM_CHANGED_IDTR
7227 | CPUM_CHANGED_TR
7228 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7229 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7230 if ( pVM->hm.s.fNestedPaging
7231 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7232 {
7233 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7234 }
7235
7236 Assert(!pVCpu->hm.s.fClearTrapFlag);
7237
7238 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7239 if (rcExit != VINF_EM_RAW_INTERRUPT)
7240 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7241
7242 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7243
7244 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7245 VMMRZCallRing3RemoveNotification(pVCpu);
7246 VMMRZCallRing3Enable(pVCpu);
7247
7248 return rc;
7249}
7250
7251
7252/**
7253 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7254 * longjump to ring-3 and possibly get preempted.
7255 *
7256 * @returns VBox status code.
7257 * @param pVCpu Pointer to the VMCPU.
7258 * @param enmOperation The operation causing the ring-3 longjump.
7259 * @param pvUser Opaque pointer to the guest-CPU context. The data
7260 * may be out-of-sync. Make sure to update the required
7261 * fields before using them.
7262 */
7263DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7264{
7265 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7266 {
7267 /*
7268 * !!! IMPORTANT !!!
7269 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7270 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7271 */
7272 VMMRZCallRing3RemoveNotification(pVCpu);
7273 VMMRZCallRing3Disable(pVCpu);
7274 HM_DISABLE_PREEMPT_IF_NEEDED();
7275
7276 PVM pVM = pVCpu->CTX_SUFF(pVM);
7277 if (CPUMIsGuestFPUStateActive(pVCpu))
7278 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7279
7280 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7281
7282#if HC_ARCH_BITS == 64
7283 /* Restore host-state bits that VT-x only restores partially. */
7284 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7285 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7286 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7287 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7288
7289 /* Restore the host MSRs as we're leaving VT-x context. */
7290 if ( pVM->hm.s.fAllow64BitGuests
7291 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7292 {
7293 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7294 }
7295#endif
7296 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7297 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7298 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7299 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7300 {
7301 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7302 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7303 }
7304
7305 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7306 VMMR0ThreadCtxHooksDeregister(pVCpu);
7307
7308 HMR0LeaveCpu(pVCpu);
7309 HM_RESTORE_PREEMPT_IF_NEEDED();
7310 return VINF_SUCCESS;
7311 }
7312
7313 Assert(pVCpu);
7314 Assert(pvUser);
7315 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7316 HMVMX_ASSERT_PREEMPT_SAFE();
7317
7318 VMMRZCallRing3Disable(pVCpu);
7319 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7320
7321 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7322 enmOperation));
7323
7324 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7325 AssertRCReturn(rc, rc);
7326
7327 VMMRZCallRing3Enable(pVCpu);
7328 return VINF_SUCCESS;
7329}
7330
7331
7332/**
7333 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7334 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7335 *
7336 * @param pVCpu Pointer to the VMCPU.
7337 */
7338DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7339{
7340 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7341 {
7342 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7343 {
7344 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7345 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7346 AssertRC(rc);
7347 Log4(("Setup interrupt-window exiting\n"));
7348 }
7349 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7350}
7351
7352
7353/**
7354 * Clears the interrupt-window exiting control in the VMCS.
7355 *
7356 * @param pVCpu Pointer to the VMCPU.
7357 */
7358DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7359{
7360 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7361 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7362 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7363 AssertRC(rc);
7364 Log4(("Cleared interrupt-window exiting\n"));
7365}
7366
7367
7368/**
7369 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7370 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7371 *
7372 * @param pVCpu Pointer to the VMCPU.
7373 */
7374DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7375{
7376 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7377 {
7378 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7379 {
7380 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7381 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7382 AssertRC(rc);
7383 Log4(("Setup NMI-window exiting\n"));
7384 }
7385 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7386}
7387
7388
7389/**
7390 * Clears the NMI-window exiting control in the VMCS.
7391 *
7392 * @param pVCpu Pointer to the VMCPU.
7393 */
7394DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7395{
7396 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7397 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7398 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7399 AssertRC(rc);
7400 Log4(("Cleared NMI-window exiting\n"));
7401}
7402
7403
7404/**
7405 * Evaluates the event to be delivered to the guest and sets it as the pending
7406 * event.
7407 *
7408 * @param pVCpu Pointer to the VMCPU.
7409 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7410 * out-of-sync. Make sure to update the required fields
7411 * before using them.
7412 */
7413static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7414{
7415 Assert(!pVCpu->hm.s.Event.fPending);
7416
7417 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7418 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7419 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7420 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7421 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7422
7423 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7424 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7425 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7426 Assert(!TRPMHasTrap(pVCpu));
7427
7428 /** @todo SMI. SMIs take priority over NMIs. */
7429 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7430 {
7431 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7432 if ( !fBlockNmi
7433 && !fBlockSti
7434 && !fBlockMovSS)
7435 {
7436 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7437 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7438 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7439
7440 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7441 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7442 }
7443 else
7444 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7445 }
7446 /*
7447 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7448 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7449 */
7450 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7451 && !pVCpu->hm.s.fSingleInstruction)
7452 {
7453 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7454 AssertRC(rc);
7455 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7456 if ( !fBlockInt
7457 && !fBlockSti
7458 && !fBlockMovSS)
7459 {
7460 uint8_t u8Interrupt;
7461 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7462 if (RT_SUCCESS(rc))
7463 {
7464 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7465 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7466 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7467
7468 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7469 }
7470 else
7471 {
7472 /** @todo Does this actually happen? If not turn it into an assertion. */
7473 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7474 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7475 }
7476 }
7477 else
7478 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7479 }
7480}
7481
7482
7483/**
7484 * Sets a pending-debug exception to be delivered to the guest if the guest is
7485 * single-stepping.
7486 *
7487 * @param pVCpu Pointer to the VMCPU.
7488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7489 * out-of-sync. Make sure to update the required fields
7490 * before using them.
7491 */
7492DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7493{
7494 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7495 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7496 {
7497 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7498 AssertRC(rc);
7499 }
7500}
7501
7502
7503/**
7504 * Injects any pending events into the guest if the guest is in a state to
7505 * receive them.
7506 *
7507 * @returns VBox status code (informational status codes included).
7508 * @param pVCpu Pointer to the VMCPU.
7509 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7510 * out-of-sync. Make sure to update the required fields
7511 * before using them.
7512 */
7513static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7514{
7515 HMVMX_ASSERT_PREEMPT_SAFE();
7516 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7517
7518 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7519 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7520 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7521 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7522
7523 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7524 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7525 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7526 Assert(!TRPMHasTrap(pVCpu));
7527
7528 int rc = VINF_SUCCESS;
7529 if (pVCpu->hm.s.Event.fPending)
7530 {
7531 /*
7532 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7533 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7534 * ended up enabling interrupts outside VT-x.
7535 */
7536 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7537 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7538 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7539 {
7540 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7541 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7542 }
7543
7544#ifdef VBOX_STRICT
7545 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7546 {
7547 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7548 if (fBlockInt)
7549 return VERR_VMX_IPE_4;
7550 Assert(!fBlockSti);
7551 Assert(!fBlockMovSS);
7552 }
7553 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7554 {
7555 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7556 Assert(!fBlockSti);
7557 Assert(!fBlockMovSS);
7558 Assert(!fBlockNmi);
7559 }
7560#endif
7561 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7562 (uint8_t)uIntType));
7563 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7564 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7565 AssertRCReturn(rc, rc);
7566
7567 /* Update the interruptibility-state as it could have been changed by
7568 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7569 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7570 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7571
7572#ifdef VBOX_WITH_STATISTICS
7573 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7574 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7575 else
7576 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7577#endif
7578 }
7579
7580 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7581 if ( fBlockSti
7582 || fBlockMovSS)
7583 {
7584 if ( !pVCpu->hm.s.fSingleInstruction
7585 && !DBGFIsStepping(pVCpu))
7586 {
7587 /*
7588 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7589 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7590 * See Intel spec. 27.3.4 "Saving Non-Register State".
7591 */
7592 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7593 AssertRCReturn(rc2, rc2);
7594 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7595 }
7596 else if (pMixedCtx->eflags.Bits.u1TF)
7597 {
7598 /*
7599 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7600 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7601 */
7602 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7603 uIntrState = 0;
7604 }
7605 }
7606
7607 /*
7608 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7609 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7610 */
7611 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7612 AssertRC(rc2);
7613
7614 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7615 NOREF(fBlockMovSS); NOREF(fBlockSti);
7616 return rc;
7617}
7618
7619
7620/**
7621 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7622 *
7623 * @param pVCpu Pointer to the VMCPU.
7624 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7625 * out-of-sync. Make sure to update the required fields
7626 * before using them.
7627 */
7628DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7629{
7630 NOREF(pMixedCtx);
7631 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7632 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7633}
7634
7635
7636/**
7637 * Injects a double-fault (#DF) exception into the VM.
7638 *
7639 * @returns VBox status code (informational status code included).
7640 * @param pVCpu Pointer to the VMCPU.
7641 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7642 * out-of-sync. Make sure to update the required fields
7643 * before using them.
7644 */
7645DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7646{
7647 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7648 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7649 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7650 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7651 puIntrState);
7652}
7653
7654
7655/**
7656 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7657 *
7658 * @param pVCpu Pointer to the VMCPU.
7659 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7660 * out-of-sync. Make sure to update the required fields
7661 * before using them.
7662 */
7663DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7664{
7665 NOREF(pMixedCtx);
7666 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7667 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7668 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7669}
7670
7671
7672/**
7673 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7674 *
7675 * @param pVCpu Pointer to the VMCPU.
7676 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7677 * out-of-sync. Make sure to update the required fields
7678 * before using them.
7679 * @param cbInstr The value of RIP that is to be pushed on the guest
7680 * stack.
7681 */
7682DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7683{
7684 NOREF(pMixedCtx);
7685 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7686 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7687 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7688}
7689
7690
7691/**
7692 * Injects a general-protection (#GP) fault into the VM.
7693 *
7694 * @returns VBox status code (informational status code included).
7695 * @param pVCpu Pointer to the VMCPU.
7696 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7697 * out-of-sync. Make sure to update the required fields
7698 * before using them.
7699 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7700 * mode, i.e. in real-mode it's not valid).
7701 * @param u32ErrorCode The error code associated with the #GP.
7702 */
7703DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7704 uint32_t *puIntrState)
7705{
7706 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7707 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7708 if (fErrorCodeValid)
7709 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7710 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7711 puIntrState);
7712}
7713
7714
7715/**
7716 * Sets a general-protection (#GP) exception as pending-for-injection into the
7717 * VM.
7718 *
7719 * @param pVCpu Pointer to the VMCPU.
7720 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7721 * out-of-sync. Make sure to update the required fields
7722 * before using them.
7723 * @param u32ErrorCode The error code associated with the #GP.
7724 */
7725DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7726{
7727 NOREF(pMixedCtx);
7728 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7729 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7730 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7731 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7732}
7733
7734
7735/**
7736 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7737 *
7738 * @param pVCpu Pointer to the VMCPU.
7739 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7740 * out-of-sync. Make sure to update the required fields
7741 * before using them.
7742 * @param uVector The software interrupt vector number.
7743 * @param cbInstr The value of RIP that is to be pushed on the guest
7744 * stack.
7745 */
7746DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7747{
7748 NOREF(pMixedCtx);
7749 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7750 if ( uVector == X86_XCPT_BP
7751 || uVector == X86_XCPT_OF)
7752 {
7753 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7754 }
7755 else
7756 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7757 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7758}
7759
7760
7761/**
7762 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7763 * stack.
7764 *
7765 * @returns VBox status code (information status code included).
7766 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7767 * @param pVM Pointer to the VM.
7768 * @param pMixedCtx Pointer to the guest-CPU context.
7769 * @param uValue The value to push to the guest stack.
7770 */
7771DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7772{
7773 /*
7774 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7775 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7776 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7777 */
7778 if (pMixedCtx->sp == 1)
7779 return VINF_EM_RESET;
7780 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7781 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7782 AssertRCReturn(rc, rc);
7783 return rc;
7784}
7785
7786
7787/**
7788 * Injects an event into the guest upon VM-entry by updating the relevant fields
7789 * in the VM-entry area in the VMCS.
7790 *
7791 * @returns VBox status code (informational error codes included).
7792 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7793 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7794 *
7795 * @param pVCpu Pointer to the VMCPU.
7796 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7797 * be out-of-sync. Make sure to update the required
7798 * fields before using them.
7799 * @param u64IntInfo The VM-entry interruption-information field.
7800 * @param cbInstr The VM-entry instruction length in bytes (for
7801 * software interrupts, exceptions and privileged
7802 * software exceptions).
7803 * @param u32ErrCode The VM-entry exception error code.
7804 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7805 * @param puIntrState Pointer to the current guest interruptibility-state.
7806 * This interruptibility-state will be updated if
7807 * necessary. This cannot not be NULL.
7808 *
7809 * @remarks Requires CR0!
7810 * @remarks No-long-jump zone!!!
7811 */
7812static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7813 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7814{
7815 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7816 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7817 Assert(puIntrState);
7818 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7819
7820 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7821 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7822
7823#ifdef VBOX_STRICT
7824 /* Validate the error-code-valid bit for hardware exceptions. */
7825 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7826 {
7827 switch (uVector)
7828 {
7829 case X86_XCPT_PF:
7830 case X86_XCPT_DF:
7831 case X86_XCPT_TS:
7832 case X86_XCPT_NP:
7833 case X86_XCPT_SS:
7834 case X86_XCPT_GP:
7835 case X86_XCPT_AC:
7836 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7837 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7838 /* fallthru */
7839 default:
7840 break;
7841 }
7842 }
7843#endif
7844
7845 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7846 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7847 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7848
7849 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7850
7851 /* We require CR0 to check if the guest is in real-mode. */
7852 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7853 AssertRCReturn(rc, rc);
7854
7855 /*
7856 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7857 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7858 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7859 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7860 */
7861 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7862 {
7863 PVM pVM = pVCpu->CTX_SUFF(pVM);
7864 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7865 {
7866 Assert(PDMVmmDevHeapIsEnabled(pVM));
7867 Assert(pVM->hm.s.vmx.pRealModeTSS);
7868
7869 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7870 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7871 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7872 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7873 AssertRCReturn(rc, rc);
7874 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7875
7876 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7877 size_t const cbIdtEntry = sizeof(X86IDTR16);
7878 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7879 {
7880 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7881 if (uVector == X86_XCPT_DF)
7882 return VINF_EM_RESET;
7883 else if (uVector == X86_XCPT_GP)
7884 {
7885 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7886 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7887 }
7888
7889 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7890 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7891 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7892 }
7893
7894 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7895 uint16_t uGuestIp = pMixedCtx->ip;
7896 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7897 {
7898 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7899 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7900 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7901 }
7902 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7903 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7904
7905 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7906 X86IDTR16 IdtEntry;
7907 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7908 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7909 AssertRCReturn(rc, rc);
7910
7911 /* Construct the stack frame for the interrupt/exception handler. */
7912 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7913 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7914 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7915 AssertRCReturn(rc, rc);
7916
7917 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7918 if (rc == VINF_SUCCESS)
7919 {
7920 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7921 pMixedCtx->rip = IdtEntry.offSel;
7922 pMixedCtx->cs.Sel = IdtEntry.uSel;
7923 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7924 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7925 && uVector == X86_XCPT_PF)
7926 {
7927 pMixedCtx->cr2 = GCPtrFaultAddress;
7928 }
7929
7930 /* If any other guest-state bits are changed here, make sure to update
7931 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7932 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7933 | HM_CHANGED_GUEST_RIP
7934 | HM_CHANGED_GUEST_RFLAGS
7935 | HM_CHANGED_GUEST_RSP);
7936
7937 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7938 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7939 {
7940 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7941 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7942 Log4(("Clearing inhibition due to STI.\n"));
7943 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7944 }
7945 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7946
7947 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7948 it, if we are returning to ring-3 before executing guest code. */
7949 pVCpu->hm.s.Event.fPending = false;
7950 }
7951 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7952 return rc;
7953 }
7954 else
7955 {
7956 /*
7957 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7958 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7959 */
7960 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7961 }
7962 }
7963
7964 /* Validate. */
7965 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7966 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7967 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7968
7969 /* Inject. */
7970 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7971 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7972 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7973 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7974
7975 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7976 && uVector == X86_XCPT_PF)
7977 {
7978 pMixedCtx->cr2 = GCPtrFaultAddress;
7979 }
7980
7981 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7982 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7983
7984 AssertRCReturn(rc, rc);
7985 return rc;
7986}
7987
7988
7989/**
7990 * Clears the interrupt-window exiting control in the VMCS and if necessary
7991 * clears the current event in the VMCS as well.
7992 *
7993 * @returns VBox status code.
7994 * @param pVCpu Pointer to the VMCPU.
7995 *
7996 * @remarks Use this function only to clear events that have not yet been
7997 * delivered to the guest but are injected in the VMCS!
7998 * @remarks No-long-jump zone!!!
7999 */
8000static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
8001{
8002 int rc;
8003 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8004
8005 /* Clear interrupt-window exiting control. */
8006 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8007 {
8008 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8009 Assert(!pVCpu->hm.s.Event.fPending);
8010 }
8011
8012 /* Clear NMI-window exiting control. */
8013 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8014 {
8015 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8016 Assert(!pVCpu->hm.s.Event.fPending);
8017 }
8018
8019 if (!pVCpu->hm.s.Event.fPending)
8020 return;
8021
8022#ifdef VBOX_STRICT
8023 uint32_t u32EntryInfo;
8024 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8025 AssertRC(rc);
8026 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8027#endif
8028
8029 /* Clear the entry-interruption field (including the valid bit). */
8030 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8031 AssertRC(rc);
8032
8033 /* Clear the pending debug exception field. */
8034 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8035 AssertRC(rc);
8036
8037 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8038 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8039}
8040
8041
8042/**
8043 * Enters the VT-x session.
8044 *
8045 * @returns VBox status code.
8046 * @param pVM Pointer to the VM.
8047 * @param pVCpu Pointer to the VMCPU.
8048 * @param pCpu Pointer to the CPU info struct.
8049 */
8050VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8051{
8052 AssertPtr(pVM);
8053 AssertPtr(pVCpu);
8054 Assert(pVM->hm.s.vmx.fSupported);
8055 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8056 NOREF(pCpu); NOREF(pVM);
8057
8058 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8059 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8060
8061#ifdef VBOX_STRICT
8062 /* Make sure we're in VMX root mode. */
8063 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8064 if (!(u32HostCR4 & X86_CR4_VMXE))
8065 {
8066 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8067 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8068 }
8069#endif
8070
8071 /*
8072 * Load the VCPU's VMCS as the current (and active) one.
8073 */
8074 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8075 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8076 if (RT_FAILURE(rc))
8077 return rc;
8078
8079 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8080 pVCpu->hm.s.fLeaveDone = false;
8081 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8082
8083 return VINF_SUCCESS;
8084}
8085
8086
8087/**
8088 * The thread-context callback (only on platforms which support it).
8089 *
8090 * @param enmEvent The thread-context event.
8091 * @param pVCpu Pointer to the VMCPU.
8092 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8093 * @thread EMT(pVCpu)
8094 */
8095VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8096{
8097 NOREF(fGlobalInit);
8098
8099 switch (enmEvent)
8100 {
8101 case RTTHREADCTXEVENT_PREEMPTING:
8102 {
8103 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8104 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8105 VMCPU_ASSERT_EMT(pVCpu);
8106
8107 PVM pVM = pVCpu->CTX_SUFF(pVM);
8108 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8109
8110 /* No longjmps (logger flushes, locks) in this fragile context. */
8111 VMMRZCallRing3Disable(pVCpu);
8112 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8113
8114 /*
8115 * Restore host-state (FPU, debug etc.)
8116 */
8117 if (!pVCpu->hm.s.fLeaveDone)
8118 {
8119 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8120 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8121 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8122 pVCpu->hm.s.fLeaveDone = true;
8123 }
8124
8125 /* Leave HM context, takes care of local init (term). */
8126 int rc = HMR0LeaveCpu(pVCpu);
8127 AssertRC(rc); NOREF(rc);
8128
8129 /* Restore longjmp state. */
8130 VMMRZCallRing3Enable(pVCpu);
8131 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8132 break;
8133 }
8134
8135 case RTTHREADCTXEVENT_RESUMED:
8136 {
8137 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8138 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8139 VMCPU_ASSERT_EMT(pVCpu);
8140
8141 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8142 VMMRZCallRing3Disable(pVCpu);
8143 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8144
8145 /* Initialize the bare minimum state required for HM. This takes care of
8146 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8147 int rc = HMR0EnterCpu(pVCpu);
8148 AssertRC(rc);
8149 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8150
8151 /* Load the active VMCS as the current one. */
8152 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8153 {
8154 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8155 AssertRC(rc); NOREF(rc);
8156 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8157 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8158 }
8159 pVCpu->hm.s.fLeaveDone = false;
8160
8161 /* Restore longjmp state. */
8162 VMMRZCallRing3Enable(pVCpu);
8163 break;
8164 }
8165
8166 default:
8167 break;
8168 }
8169}
8170
8171
8172/**
8173 * Saves the host state in the VMCS host-state.
8174 * Sets up the VM-exit MSR-load area.
8175 *
8176 * The CPU state will be loaded from these fields on every successful VM-exit.
8177 *
8178 * @returns VBox status code.
8179 * @param pVM Pointer to the VM.
8180 * @param pVCpu Pointer to the VMCPU.
8181 *
8182 * @remarks No-long-jump zone!!!
8183 */
8184static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8185{
8186 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8187
8188 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8189 return VINF_SUCCESS;
8190
8191 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8192 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8193
8194 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8195 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8196
8197 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8198 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8199
8200 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8201 return rc;
8202}
8203
8204
8205/**
8206 * Saves the host state in the VMCS host-state.
8207 *
8208 * @returns VBox status code.
8209 * @param pVM Pointer to the VM.
8210 * @param pVCpu Pointer to the VMCPU.
8211 *
8212 * @remarks No-long-jump zone!!!
8213 */
8214VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8215{
8216 AssertPtr(pVM);
8217 AssertPtr(pVCpu);
8218
8219 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8220
8221 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8222 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8223 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8224 return hmR0VmxSaveHostState(pVM, pVCpu);
8225}
8226
8227
8228/**
8229 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8230 * loaded from these fields on every successful VM-entry.
8231 *
8232 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8233 * Sets up the VM-entry controls.
8234 * Sets up the appropriate VMX non-root function to execute guest code based on
8235 * the guest CPU mode.
8236 *
8237 * @returns VBox status code.
8238 * @param pVM Pointer to the VM.
8239 * @param pVCpu Pointer to the VMCPU.
8240 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8241 * out-of-sync. Make sure to update the required fields
8242 * before using them.
8243 *
8244 * @remarks No-long-jump zone!!!
8245 */
8246static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8247{
8248 AssertPtr(pVM);
8249 AssertPtr(pVCpu);
8250 AssertPtr(pMixedCtx);
8251 HMVMX_ASSERT_PREEMPT_SAFE();
8252
8253#ifdef LOG_ENABLED
8254 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8255 * probably not initialized yet? Anyway this will do for now.
8256 *
8257 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8258 * interface and disable ring-3 calls when thread-context hooks are not
8259 * available. */
8260 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8261 VMMR0LogFlushDisable(pVCpu);
8262#endif
8263
8264 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8265
8266 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8267
8268 /* Determine real-on-v86 mode. */
8269 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8270 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8271 && CPUMIsGuestInRealModeEx(pMixedCtx))
8272 {
8273 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8274 }
8275
8276 /*
8277 * Load the guest-state into the VMCS.
8278 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8279 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8280 */
8281 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8282 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8283
8284 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8285 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8289 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8290 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8291
8292 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8293 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8294
8295 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8296 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8297
8298 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8299 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8300 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8301
8302 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8303 determine we don't have to swap EFER after all. */
8304 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8305 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8306
8307 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8308 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8309
8310 /*
8311 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8312 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8313 */
8314 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8315 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8316
8317 /* Clear any unused and reserved bits. */
8318 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8319
8320#ifdef LOG_ENABLED
8321 /* Only reenable log-flushing if the caller has it enabled. */
8322 if (!fCallerDisabledLogFlush)
8323 VMMR0LogFlushEnable(pVCpu);
8324#endif
8325
8326 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8327 return rc;
8328}
8329
8330
8331/**
8332 * Loads the state shared between the host and guest into the VMCS.
8333 *
8334 * @param pVM Pointer to the VM.
8335 * @param pVCpu Pointer to the VMCPU.
8336 * @param pCtx Pointer to the guest-CPU context.
8337 *
8338 * @remarks No-long-jump zone!!!
8339 */
8340static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8341{
8342 NOREF(pVM);
8343
8344 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8345 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8346
8347 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8348 {
8349 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8350 AssertRC(rc);
8351 }
8352
8353 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8354 {
8355 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8356 AssertRC(rc);
8357
8358 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8360 {
8361 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8362 AssertRC(rc);
8363 }
8364 }
8365
8366 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8367 {
8368#if HC_ARCH_BITS == 64
8369 if (pVM->hm.s.fAllow64BitGuests)
8370 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8371#endif
8372 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8373 }
8374
8375 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8376 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8377}
8378
8379
8380/**
8381 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8382 *
8383 * @param pVM Pointer to the VM.
8384 * @param pVCpu Pointer to the VMCPU.
8385 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8386 * out-of-sync. Make sure to update the required fields
8387 * before using them.
8388 */
8389DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8390{
8391 HMVMX_ASSERT_PREEMPT_SAFE();
8392
8393 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8394#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8395 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8396#endif
8397
8398 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8399 {
8400 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8401 AssertRC(rc);
8402 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8403 }
8404 else if (HMCPU_CF_VALUE(pVCpu))
8405 {
8406 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8407 AssertRC(rc);
8408 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8409 }
8410
8411 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8412 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8413 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8414 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8415}
8416
8417
8418/**
8419 * Does the preparations before executing guest code in VT-x.
8420 *
8421 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8422 * recompiler. We must be cautious what we do here regarding committing
8423 * guest-state information into the VMCS assuming we assuredly execute the
8424 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8425 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8426 * so that the recompiler can (and should) use them when it resumes guest
8427 * execution. Otherwise such operations must be done when we can no longer
8428 * exit to ring-3.
8429 *
8430 * @returns Strict VBox status code.
8431 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8432 * have been disabled.
8433 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8434 * double-fault into the guest.
8435 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8436 *
8437 * @param pVM Pointer to the VM.
8438 * @param pVCpu Pointer to the VMCPU.
8439 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8440 * out-of-sync. Make sure to update the required fields
8441 * before using them.
8442 * @param pVmxTransient Pointer to the VMX transient structure.
8443 */
8444static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8445{
8446 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8447
8448#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8449 PGMRZDynMapFlushAutoSet(pVCpu);
8450#endif
8451
8452 /* Check force flag actions that might require us to go back to ring-3. */
8453 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8454 if (rc != VINF_SUCCESS)
8455 return rc;
8456
8457#ifndef IEM_VERIFICATION_MODE_FULL
8458 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8459 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8460 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8461 {
8462 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8463 RTGCPHYS GCPhysApicBase;
8464 GCPhysApicBase = pMixedCtx->msrApicBase;
8465 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8466
8467 /* Unalias any existing mapping. */
8468 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8469 AssertRCReturn(rc, rc);
8470
8471 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8472 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8473 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8474 AssertRCReturn(rc, rc);
8475
8476 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8477 }
8478#endif /* !IEM_VERIFICATION_MODE_FULL */
8479
8480 /*
8481 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8482 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8483 */
8484 if (TRPMHasTrap(pVCpu))
8485 {
8486 rc = hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8487 if (RT_FAILURE(rc))
8488 return rc;
8489 }
8490 else if (!pVCpu->hm.s.Event.fPending)
8491 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8492
8493 /*
8494 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8495 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8496 */
8497 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8498 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8499 {
8500 //Assert(rc == VINF_EM_RESET);
8501 return rc;
8502 }
8503
8504 /*
8505 * Load the guest state bits, we can handle longjmps/getting preempted here.
8506 *
8507 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8508 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8509 * Hence, this needs to be done -after- injection of events.
8510 */
8511 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8512
8513 /*
8514 * No longjmps to ring-3 from this point on!!!
8515 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8516 * This also disables flushing of the R0-logger instance (if any).
8517 */
8518 VMMRZCallRing3Disable(pVCpu);
8519
8520 /*
8521 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8522 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8523 *
8524 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8525 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8526 *
8527 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8528 * executing guest code.
8529 */
8530 pVmxTransient->uEflags = ASMIntDisableFlags();
8531 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8532 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8533 {
8534 hmR0VmxClearEventVmcs(pVCpu);
8535 ASMSetFlags(pVmxTransient->uEflags);
8536 VMMRZCallRing3Enable(pVCpu);
8537 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8538 return VINF_EM_RAW_TO_R3;
8539 }
8540
8541 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8542 {
8543 hmR0VmxClearEventVmcs(pVCpu);
8544 ASMSetFlags(pVmxTransient->uEflags);
8545 VMMRZCallRing3Enable(pVCpu);
8546 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8547 return VINF_EM_RAW_INTERRUPT;
8548 }
8549
8550 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8551 pVCpu->hm.s.Event.fPending = false;
8552
8553 return VINF_SUCCESS;
8554}
8555
8556
8557/**
8558 * Prepares to run guest code in VT-x and we've committed to doing so. This
8559 * means there is no backing out to ring-3 or anywhere else at this
8560 * point.
8561 *
8562 * @param pVM Pointer to the VM.
8563 * @param pVCpu Pointer to the VMCPU.
8564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8565 * out-of-sync. Make sure to update the required fields
8566 * before using them.
8567 * @param pVmxTransient Pointer to the VMX transient structure.
8568 *
8569 * @remarks Called with preemption disabled.
8570 * @remarks No-long-jump zone!!!
8571 */
8572static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8573{
8574 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8575 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8576 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8577
8578 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8579 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8580
8581#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8582 if (!CPUMIsGuestFPUStateActive(pVCpu))
8583 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8584 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8585#endif
8586
8587 if ( pVCpu->hm.s.fUseGuestFpu
8588 && !CPUMIsGuestFPUStateActive(pVCpu))
8589 {
8590 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8591 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8592 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8593 }
8594
8595 /*
8596 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8597 */
8598 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8599 && pVCpu->hm.s.vmx.cMsrs > 0)
8600 {
8601 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8602 }
8603
8604 /*
8605 * Load the host state bits as we may've been preempted (only happens when
8606 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8607 */
8608 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8609 {
8610 /* This ASSUMES that pfnStartVM has been set up already. */
8611 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8612 AssertRC(rc);
8613 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8614 }
8615 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8616
8617 /*
8618 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8619 */
8620 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8621 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8622 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8623
8624 /* Store status of the shared guest-host state at the time of VM-entry. */
8625#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8626 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8627 {
8628 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8629 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8630 }
8631 else
8632#endif
8633 {
8634 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8635 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8636 }
8637 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8638
8639 /*
8640 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8641 */
8642 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8643 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8644
8645 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8646 RTCPUID idCurrentCpu = pCpu->idCpu;
8647 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8648 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8649 {
8650 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8651 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8652 }
8653
8654 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8655 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8656 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8657 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8658
8659 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8660
8661 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8662 to start executing. */
8663
8664 /*
8665 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8666 */
8667 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8668 {
8669 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8670 {
8671 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8672 AssertRC(rc2);
8673 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8674 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8675 true /* fUpdateHostMsr */);
8676 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8677 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8678 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8679 }
8680 else
8681 {
8682 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8683 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8684 }
8685 }
8686
8687#ifdef VBOX_STRICT
8688 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8689 hmR0VmxCheckHostEferMsr(pVCpu);
8690 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8691#endif
8692#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8693 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8694 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8695 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8696#endif
8697}
8698
8699
8700/**
8701 * Performs some essential restoration of state after running guest code in
8702 * VT-x.
8703 *
8704 * @param pVM Pointer to the VM.
8705 * @param pVCpu Pointer to the VMCPU.
8706 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8707 * out-of-sync. Make sure to update the required fields
8708 * before using them.
8709 * @param pVmxTransient Pointer to the VMX transient structure.
8710 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8711 *
8712 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8713 *
8714 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8715 * unconditionally when it is safe to do so.
8716 */
8717static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8718{
8719 NOREF(pVM);
8720
8721 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8722
8723 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8724 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8725 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8726 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8727 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8728
8729 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8730 {
8731 /** @todo Find a way to fix hardcoding a guestimate. */
8732 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8733 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8734 }
8735
8736 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8737 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8738 Assert(!(ASMGetFlags() & X86_EFL_IF));
8739 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8740
8741#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8742 if (CPUMIsGuestFPUStateActive(pVCpu))
8743 {
8744 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8745 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8746 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8747 }
8748#endif
8749
8750#if HC_ARCH_BITS == 64
8751 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8752#endif
8753 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8754#ifdef VBOX_STRICT
8755 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8756#endif
8757 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8758 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8759
8760 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8761 uint32_t uExitReason;
8762 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8763 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8764 AssertRC(rc);
8765 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8766 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8767
8768 /* Update the VM-exit history array. */
8769 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8770
8771 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8772 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8773 {
8774 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8775 pVmxTransient->fVMEntryFailed));
8776 return;
8777 }
8778
8779 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8780 {
8781 /** @todo We can optimize this by only syncing with our force-flags when
8782 * really needed and keeping the VMCS state as it is for most
8783 * VM-exits. */
8784 /* Update the guest interruptibility-state from the VMCS. */
8785 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8786
8787#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8788 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8789 AssertRC(rc);
8790#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8791 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8792 AssertRC(rc);
8793#endif
8794
8795 /*
8796 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8797 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8798 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8799 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8800 */
8801 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8802 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8803 {
8804 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8805 AssertRC(rc);
8806 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8807 }
8808 }
8809}
8810
8811
8812/**
8813 * Runs the guest code using VT-x the normal way.
8814 *
8815 * @returns VBox status code.
8816 * @param pVM Pointer to the VM.
8817 * @param pVCpu Pointer to the VMCPU.
8818 * @param pCtx Pointer to the guest-CPU context.
8819 *
8820 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8821 */
8822static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8823{
8824 VMXTRANSIENT VmxTransient;
8825 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8826 int rc = VERR_INTERNAL_ERROR_5;
8827 uint32_t cLoops = 0;
8828
8829 for (;; cLoops++)
8830 {
8831 Assert(!HMR0SuspendPending());
8832 HMVMX_ASSERT_CPU_SAFE();
8833
8834 /* Preparatory work for running guest code, this may force us to return
8835 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8836 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8837 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8838 if (rc != VINF_SUCCESS)
8839 break;
8840
8841 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8842 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8843 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8844
8845 /* Restore any residual host-state and save any bits shared between host
8846 and guest into the guest-CPU state. Re-enables interrupts! */
8847 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8848
8849 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8850 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8851 {
8852 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8853 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8854 return rc;
8855 }
8856
8857 /* Handle the VM-exit. */
8858 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8859 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8860 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8861 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8862 HMVMX_START_EXIT_DISPATCH_PROF();
8863#ifdef HMVMX_USE_FUNCTION_TABLE
8864 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8865#else
8866 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8867#endif
8868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8869 if (rc != VINF_SUCCESS)
8870 break;
8871 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8872 {
8873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8874 rc = VINF_EM_RAW_INTERRUPT;
8875 break;
8876 }
8877 }
8878
8879 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8880 return rc;
8881}
8882
8883
8884/**
8885 * Single steps guest code using VT-x.
8886 *
8887 * @returns VBox status code.
8888 * @param pVM Pointer to the VM.
8889 * @param pVCpu Pointer to the VMCPU.
8890 * @param pCtx Pointer to the guest-CPU context.
8891 *
8892 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8893 */
8894static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8895{
8896 VMXTRANSIENT VmxTransient;
8897 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8898 int rc = VERR_INTERNAL_ERROR_5;
8899 uint32_t cLoops = 0;
8900 uint16_t uCsStart = pCtx->cs.Sel;
8901 uint64_t uRipStart = pCtx->rip;
8902
8903 for (;; cLoops++)
8904 {
8905 Assert(!HMR0SuspendPending());
8906 HMVMX_ASSERT_CPU_SAFE();
8907
8908 /* Preparatory work for running guest code, this may force us to return
8909 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8910 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8911 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8912 if (rc != VINF_SUCCESS)
8913 break;
8914
8915 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8916 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8917 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8918
8919 /* Restore any residual host-state and save any bits shared between host
8920 and guest into the guest-CPU state. Re-enables interrupts! */
8921 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8922
8923 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8924 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8925 {
8926 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8927 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8928 return rc;
8929 }
8930
8931 /* Handle the VM-exit. */
8932 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8933 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8934 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8935 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8936 HMVMX_START_EXIT_DISPATCH_PROF();
8937#ifdef HMVMX_USE_FUNCTION_TABLE
8938 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8939#else
8940 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8941#endif
8942 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8943 if (rc != VINF_SUCCESS)
8944 break;
8945 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8946 {
8947 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8948 rc = VINF_EM_RAW_INTERRUPT;
8949 break;
8950 }
8951
8952 /*
8953 * Did the RIP change, if so, consider it a single step.
8954 * Otherwise, make sure one of the TFs gets set.
8955 */
8956 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8957 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8958 AssertRCReturn(rc2, rc2);
8959 if ( pCtx->rip != uRipStart
8960 || pCtx->cs.Sel != uCsStart)
8961 {
8962 rc = VINF_EM_DBG_STEPPED;
8963 break;
8964 }
8965 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8966 }
8967
8968 /*
8969 * Clear the X86_EFL_TF if necessary.
8970 */
8971 if (pVCpu->hm.s.fClearTrapFlag)
8972 {
8973 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8974 AssertRCReturn(rc2, rc2);
8975 pVCpu->hm.s.fClearTrapFlag = false;
8976 pCtx->eflags.Bits.u1TF = 0;
8977 }
8978 /** @todo there seems to be issues with the resume flag when the monitor trap
8979 * flag is pending without being used. Seen early in bios init when
8980 * accessing APIC page in protected mode. */
8981
8982 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8983 return rc;
8984}
8985
8986
8987/**
8988 * Runs the guest code using VT-x.
8989 *
8990 * @returns VBox status code.
8991 * @param pVM Pointer to the VM.
8992 * @param pVCpu Pointer to the VMCPU.
8993 * @param pCtx Pointer to the guest-CPU context.
8994 */
8995VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8996{
8997 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8998 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8999 HMVMX_ASSERT_PREEMPT_SAFE();
9000
9001 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
9002
9003 int rc;
9004 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
9005 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9006 else
9007 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
9008
9009 if (rc == VERR_EM_INTERPRETER)
9010 rc = VINF_EM_RAW_EMULATE_INSTR;
9011 else if (rc == VINF_EM_RESET)
9012 rc = VINF_EM_TRIPLE_FAULT;
9013
9014 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9015 if (RT_FAILURE(rc2))
9016 {
9017 pVCpu->hm.s.u32HMError = rc;
9018 rc = rc2;
9019 }
9020 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9021 return rc;
9022}
9023
9024
9025#ifndef HMVMX_USE_FUNCTION_TABLE
9026DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9027{
9028#ifdef DEBUG_ramshankar
9029# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9030# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9031#endif
9032 int rc;
9033 switch (rcReason)
9034 {
9035 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9048 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9049 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9050 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9051 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9052 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9053 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9054 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9055 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9056 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9057 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9058 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9059 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9060 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9061 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9062 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9063 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9064 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9065 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9066 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9067 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9068 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9069
9070 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9071 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9072 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9073 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9074 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9075 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9076 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9077 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9078 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9079
9080 case VMX_EXIT_VMCLEAR:
9081 case VMX_EXIT_VMLAUNCH:
9082 case VMX_EXIT_VMPTRLD:
9083 case VMX_EXIT_VMPTRST:
9084 case VMX_EXIT_VMREAD:
9085 case VMX_EXIT_VMRESUME:
9086 case VMX_EXIT_VMWRITE:
9087 case VMX_EXIT_VMXOFF:
9088 case VMX_EXIT_VMXON:
9089 case VMX_EXIT_INVEPT:
9090 case VMX_EXIT_INVVPID:
9091 case VMX_EXIT_VMFUNC:
9092 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9093 break;
9094 default:
9095 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9096 break;
9097 }
9098 return rc;
9099}
9100#endif
9101
9102#ifdef DEBUG
9103/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9104# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9105 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9106
9107# define HMVMX_ASSERT_PREEMPT_CPUID() \
9108 do \
9109 { \
9110 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9111 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9112 } while (0)
9113
9114# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9115 do { \
9116 AssertPtr(pVCpu); \
9117 AssertPtr(pMixedCtx); \
9118 AssertPtr(pVmxTransient); \
9119 Assert(pVmxTransient->fVMEntryFailed == false); \
9120 Assert(ASMIntAreEnabled()); \
9121 HMVMX_ASSERT_PREEMPT_SAFE(); \
9122 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9123 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9124 HMVMX_ASSERT_PREEMPT_SAFE(); \
9125 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9126 HMVMX_ASSERT_PREEMPT_CPUID(); \
9127 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9128 } while (0)
9129
9130# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9131 do { \
9132 Log4Func(("\n")); \
9133 } while (0)
9134#else /* Release builds */
9135# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9136 do { \
9137 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9138 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9139 } while (0)
9140# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9141#endif
9142
9143
9144/**
9145 * Advances the guest RIP after reading it from the VMCS.
9146 *
9147 * @returns VBox status code.
9148 * @param pVCpu Pointer to the VMCPU.
9149 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9150 * out-of-sync. Make sure to update the required fields
9151 * before using them.
9152 * @param pVmxTransient Pointer to the VMX transient structure.
9153 *
9154 * @remarks No-long-jump zone!!!
9155 */
9156DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9157{
9158 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9159 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9160 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9161 AssertRCReturn(rc, rc);
9162
9163 pMixedCtx->rip += pVmxTransient->cbInstr;
9164 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9165
9166 /*
9167 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9168 * pending debug exception field as it takes care of priority of events.
9169 *
9170 * See Intel spec. 32.2.1 "Debug Exceptions".
9171 */
9172 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9173
9174 return rc;
9175}
9176
9177
9178/**
9179 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9180 * and update error record fields accordingly.
9181 *
9182 * @return VMX_IGS_* return codes.
9183 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9184 * wrong with the guest state.
9185 *
9186 * @param pVM Pointer to the VM.
9187 * @param pVCpu Pointer to the VMCPU.
9188 * @param pCtx Pointer to the guest-CPU state.
9189 *
9190 * @remarks This function assumes our cache of the VMCS controls
9191 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9192 */
9193static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9194{
9195#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9196#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9197 uError = (err); \
9198 break; \
9199 } else do { } while (0)
9200
9201 int rc;
9202 uint32_t uError = VMX_IGS_ERROR;
9203 uint32_t u32Val;
9204 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9205
9206 do
9207 {
9208 /*
9209 * CR0.
9210 */
9211 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9212 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9213 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9214 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9215 if (fUnrestrictedGuest)
9216 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9217
9218 uint32_t u32GuestCR0;
9219 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9220 AssertRCBreak(rc);
9221 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9222 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9223 if ( !fUnrestrictedGuest
9224 && (u32GuestCR0 & X86_CR0_PG)
9225 && !(u32GuestCR0 & X86_CR0_PE))
9226 {
9227 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9228 }
9229
9230 /*
9231 * CR4.
9232 */
9233 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9234 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9235
9236 uint32_t u32GuestCR4;
9237 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9238 AssertRCBreak(rc);
9239 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9240 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9241
9242 /*
9243 * IA32_DEBUGCTL MSR.
9244 */
9245 uint64_t u64Val;
9246 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9247 AssertRCBreak(rc);
9248 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9249 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9250 {
9251 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9252 }
9253 uint64_t u64DebugCtlMsr = u64Val;
9254
9255#ifdef VBOX_STRICT
9256 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9257 AssertRCBreak(rc);
9258 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9259#endif
9260 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9261
9262 /*
9263 * RIP and RFLAGS.
9264 */
9265 uint32_t u32Eflags;
9266#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9267 if (HMVMX_IS_64BIT_HOST_MODE())
9268 {
9269 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9270 AssertRCBreak(rc);
9271 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9272 if ( !fLongModeGuest
9273 || !pCtx->cs.Attr.n.u1Long)
9274 {
9275 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9276 }
9277 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9278 * must be identical if the "IA-32e mode guest" VM-entry
9279 * control is 1 and CS.L is 1. No check applies if the
9280 * CPU supports 64 linear-address bits. */
9281
9282 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9283 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9284 AssertRCBreak(rc);
9285 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9286 VMX_IGS_RFLAGS_RESERVED);
9287 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9288 u32Eflags = u64Val;
9289 }
9290 else
9291#endif
9292 {
9293 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9294 AssertRCBreak(rc);
9295 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9296 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9297 }
9298
9299 if ( fLongModeGuest
9300 || ( fUnrestrictedGuest
9301 && !(u32GuestCR0 & X86_CR0_PE)))
9302 {
9303 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9304 }
9305
9306 uint32_t u32EntryInfo;
9307 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9308 AssertRCBreak(rc);
9309 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9310 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9311 {
9312 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9313 }
9314
9315 /*
9316 * 64-bit checks.
9317 */
9318#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9319 if (HMVMX_IS_64BIT_HOST_MODE())
9320 {
9321 if ( fLongModeGuest
9322 && !fUnrestrictedGuest)
9323 {
9324 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9325 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9326 }
9327
9328 if ( !fLongModeGuest
9329 && (u32GuestCR4 & X86_CR4_PCIDE))
9330 {
9331 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9332 }
9333
9334 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9335 * 51:32 beyond the processor's physical-address width are 0. */
9336
9337 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9338 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9339 {
9340 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9341 }
9342
9343 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9344 AssertRCBreak(rc);
9345 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9346
9347 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9348 AssertRCBreak(rc);
9349 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9350 }
9351#endif
9352
9353 /*
9354 * PERF_GLOBAL MSR.
9355 */
9356 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9357 {
9358 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9359 AssertRCBreak(rc);
9360 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9361 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9362 }
9363
9364 /*
9365 * PAT MSR.
9366 */
9367 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9368 {
9369 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9370 AssertRCBreak(rc);
9371 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9372 for (unsigned i = 0; i < 8; i++)
9373 {
9374 uint8_t u8Val = (u64Val & 0x7);
9375 if ( u8Val != 0 /* UC */
9376 || u8Val != 1 /* WC */
9377 || u8Val != 4 /* WT */
9378 || u8Val != 5 /* WP */
9379 || u8Val != 6 /* WB */
9380 || u8Val != 7 /* UC- */)
9381 {
9382 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9383 }
9384 u64Val >>= 3;
9385 }
9386 }
9387
9388 /*
9389 * EFER MSR.
9390 */
9391 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9392 {
9393 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9394 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9395 AssertRCBreak(rc);
9396 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9397 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9398 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9399 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9400 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9401 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9402 VMX_IGS_EFER_LMA_PG_MISMATCH);
9403 }
9404
9405 /*
9406 * Segment registers.
9407 */
9408 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9409 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9410 if (!(u32Eflags & X86_EFL_VM))
9411 {
9412 /* CS */
9413 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9414 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9415 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9416 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9417 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9418 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9419 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9420 /* CS cannot be loaded with NULL in protected mode. */
9421 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9422 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9423 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9424 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9425 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9426 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9427 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9428 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9429 else
9430 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9431
9432 /* SS */
9433 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9434 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9435 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9436 if ( !(pCtx->cr0 & X86_CR0_PE)
9437 || pCtx->cs.Attr.n.u4Type == 3)
9438 {
9439 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9440 }
9441 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9442 {
9443 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9444 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9445 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9446 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9447 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9448 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9449 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9450 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9451 }
9452
9453 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9454 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9455 {
9456 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9457 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9458 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9459 || pCtx->ds.Attr.n.u4Type > 11
9460 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9461 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9462 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9463 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9464 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9465 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9466 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9467 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9468 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9469 }
9470 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9471 {
9472 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9473 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9474 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9475 || pCtx->es.Attr.n.u4Type > 11
9476 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9477 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9478 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9479 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9480 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9481 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9482 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9483 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9484 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9485 }
9486 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9487 {
9488 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9489 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9490 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9491 || pCtx->fs.Attr.n.u4Type > 11
9492 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9493 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9494 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9495 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9496 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9497 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9498 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9499 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9500 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9501 }
9502 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9503 {
9504 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9505 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9506 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9507 || pCtx->gs.Attr.n.u4Type > 11
9508 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9509 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9510 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9511 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9512 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9513 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9514 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9515 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9516 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9517 }
9518 /* 64-bit capable CPUs. */
9519#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9520 if (HMVMX_IS_64BIT_HOST_MODE())
9521 {
9522 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9523 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9524 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9525 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9526 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9527 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9528 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9529 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9530 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9531 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9532 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9533 }
9534#endif
9535 }
9536 else
9537 {
9538 /* V86 mode checks. */
9539 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9540 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9541 {
9542 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9543 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9544 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9545 }
9546 else
9547 {
9548 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9549 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9550 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9551 }
9552
9553 /* CS */
9554 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9555 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9556 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9557 /* SS */
9558 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9559 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9560 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9561 /* DS */
9562 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9563 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9564 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9565 /* ES */
9566 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9567 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9568 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9569 /* FS */
9570 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9571 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9572 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9573 /* GS */
9574 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9575 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9576 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9577 /* 64-bit capable CPUs. */
9578#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9579 if (HMVMX_IS_64BIT_HOST_MODE())
9580 {
9581 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9582 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9583 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9584 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9585 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9586 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9587 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9588 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9589 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9590 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9591 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9592 }
9593#endif
9594 }
9595
9596 /*
9597 * TR.
9598 */
9599 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9600 /* 64-bit capable CPUs. */
9601#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9602 if (HMVMX_IS_64BIT_HOST_MODE())
9603 {
9604 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9605 }
9606#endif
9607 if (fLongModeGuest)
9608 {
9609 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9610 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9611 }
9612 else
9613 {
9614 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9615 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9616 VMX_IGS_TR_ATTR_TYPE_INVALID);
9617 }
9618 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9619 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9620 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9621 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9622 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9623 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9624 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9625 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9626
9627 /*
9628 * GDTR and IDTR.
9629 */
9630#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9631 if (HMVMX_IS_64BIT_HOST_MODE())
9632 {
9633 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9634 AssertRCBreak(rc);
9635 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9636
9637 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9638 AssertRCBreak(rc);
9639 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9640 }
9641#endif
9642
9643 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9644 AssertRCBreak(rc);
9645 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9646
9647 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9648 AssertRCBreak(rc);
9649 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9650
9651 /*
9652 * Guest Non-Register State.
9653 */
9654 /* Activity State. */
9655 uint32_t u32ActivityState;
9656 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9657 AssertRCBreak(rc);
9658 HMVMX_CHECK_BREAK( !u32ActivityState
9659 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9660 VMX_IGS_ACTIVITY_STATE_INVALID);
9661 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9662 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9663 uint32_t u32IntrState;
9664 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9665 AssertRCBreak(rc);
9666 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9667 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9668 {
9669 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9670 }
9671
9672 /** @todo Activity state and injecting interrupts. Left as a todo since we
9673 * currently don't use activity states but ACTIVE. */
9674
9675 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9676 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9677
9678 /* Guest interruptibility-state. */
9679 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9680 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9681 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9682 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9683 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9684 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9685 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9686 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9687 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9688 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9689 {
9690 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9691 {
9692 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9693 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9694 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9695 }
9696 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9697 {
9698 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9699 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9700 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9701 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9702 }
9703 }
9704 /** @todo Assumes the processor is not in SMM. */
9705 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9706 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9707 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9708 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9709 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9710 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9711 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9712 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9713 {
9714 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9715 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9716 }
9717
9718 /* Pending debug exceptions. */
9719 if (HMVMX_IS_64BIT_HOST_MODE())
9720 {
9721 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9722 AssertRCBreak(rc);
9723 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9724 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9725 u32Val = u64Val; /* For pending debug exceptions checks below. */
9726 }
9727 else
9728 {
9729 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9730 AssertRCBreak(rc);
9731 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9732 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9733 }
9734
9735 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9736 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9737 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9738 {
9739 if ( (u32Eflags & X86_EFL_TF)
9740 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9741 {
9742 /* Bit 14 is PendingDebug.BS. */
9743 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9744 }
9745 if ( !(u32Eflags & X86_EFL_TF)
9746 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9747 {
9748 /* Bit 14 is PendingDebug.BS. */
9749 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9750 }
9751 }
9752
9753 /* VMCS link pointer. */
9754 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9755 AssertRCBreak(rc);
9756 if (u64Val != UINT64_C(0xffffffffffffffff))
9757 {
9758 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9759 /** @todo Bits beyond the processor's physical-address width MBZ. */
9760 /** @todo 32-bit located in memory referenced by value of this field (as a
9761 * physical address) must contain the processor's VMCS revision ID. */
9762 /** @todo SMM checks. */
9763 }
9764
9765 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9766 * not using Nested Paging? */
9767 if ( pVM->hm.s.fNestedPaging
9768 && !fLongModeGuest
9769 && CPUMIsGuestInPAEModeEx(pCtx))
9770 {
9771 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9772 AssertRCBreak(rc);
9773 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9774
9775 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9776 AssertRCBreak(rc);
9777 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9778
9779 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9780 AssertRCBreak(rc);
9781 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9782
9783 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9784 AssertRCBreak(rc);
9785 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9786 }
9787
9788 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9789 if (uError == VMX_IGS_ERROR)
9790 uError = VMX_IGS_REASON_NOT_FOUND;
9791 } while (0);
9792
9793 pVCpu->hm.s.u32HMError = uError;
9794 return uError;
9795
9796#undef HMVMX_ERROR_BREAK
9797#undef HMVMX_CHECK_BREAK
9798}
9799
9800/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9801/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9802/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9803
9804/** @name VM-exit handlers.
9805 * @{
9806 */
9807
9808/**
9809 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9810 */
9811HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9812{
9813 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9814 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9815 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9816 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9817 return VINF_SUCCESS;
9818 return VINF_EM_RAW_INTERRUPT;
9819}
9820
9821
9822/**
9823 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9824 */
9825HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9826{
9827 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9828 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9829
9830 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9831 AssertRCReturn(rc, rc);
9832
9833 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9834 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9835 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9836 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9837
9838 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9839 {
9840 /*
9841 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9842 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9843 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9844 *
9845 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9846 */
9847 VMXDispatchHostNmi();
9848 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9849 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9850 return VINF_SUCCESS;
9851 }
9852
9853 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9854 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9855 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9856 {
9857 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9858 return VINF_SUCCESS;
9859 }
9860 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9861 {
9862 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9863 return rc;
9864 }
9865
9866 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9867 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9868 switch (uIntType)
9869 {
9870 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9871 Assert(uVector == X86_XCPT_DB);
9872 /* no break */
9873 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9874 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9875 /* no break */
9876 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9877 {
9878 switch (uVector)
9879 {
9880 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9881 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9882 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9883 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9884 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9885 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9886#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9887 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9888 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9889 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9890 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9891 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9892 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9893 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9894 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9895 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9896 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9897 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9898 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9899#endif
9900 default:
9901 {
9902 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9903 AssertRCReturn(rc, rc);
9904
9905 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9906 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9907 {
9908 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9909 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9910 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9911
9912 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9913 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9914 AssertRCReturn(rc, rc);
9915 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9916 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9917 0 /* GCPtrFaultAddress */);
9918 AssertRCReturn(rc, rc);
9919 }
9920 else
9921 {
9922 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9923 pVCpu->hm.s.u32HMError = uVector;
9924 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9925 }
9926 break;
9927 }
9928 }
9929 break;
9930 }
9931
9932 default:
9933 {
9934 pVCpu->hm.s.u32HMError = uExitIntInfo;
9935 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9936 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9937 break;
9938 }
9939 }
9940 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9941 return rc;
9942}
9943
9944
9945/**
9946 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9947 */
9948HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9949{
9950 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9951
9952 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9953 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9954
9955 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9956 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9957 return VINF_SUCCESS;
9958}
9959
9960
9961/**
9962 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9963 */
9964HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9965{
9966 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9967 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9968 {
9969 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9970 HMVMX_RETURN_UNEXPECTED_EXIT();
9971 }
9972
9973 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9974
9975 /*
9976 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9977 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9978 */
9979 uint32_t uIntrState = 0;
9980 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9981 AssertRCReturn(rc, rc);
9982
9983 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9984 if ( fBlockSti
9985 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9986 {
9987 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9988 }
9989
9990 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9991 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9992
9993 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9994 return VINF_SUCCESS;
9995}
9996
9997
9998/**
9999 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10000 */
10001HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10002{
10003 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10004 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10005 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10006}
10007
10008
10009/**
10010 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10011 */
10012HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10013{
10014 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10015 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10016 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10017}
10018
10019
10020/**
10021 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10022 */
10023HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10024{
10025 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10026 PVM pVM = pVCpu->CTX_SUFF(pVM);
10027 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10028 if (RT_LIKELY(rc == VINF_SUCCESS))
10029 {
10030 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10031 Assert(pVmxTransient->cbInstr == 2);
10032 }
10033 else
10034 {
10035 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10036 rc = VERR_EM_INTERPRETER;
10037 }
10038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10039 return rc;
10040}
10041
10042
10043/**
10044 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10045 */
10046HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10047{
10048 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10049 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10050 AssertRCReturn(rc, rc);
10051
10052 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10053 return VINF_EM_RAW_EMULATE_INSTR;
10054
10055 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10056 HMVMX_RETURN_UNEXPECTED_EXIT();
10057}
10058
10059
10060/**
10061 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10062 */
10063HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10064{
10065 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10066 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10067 AssertRCReturn(rc, rc);
10068
10069 PVM pVM = pVCpu->CTX_SUFF(pVM);
10070 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10071 if (RT_LIKELY(rc == VINF_SUCCESS))
10072 {
10073 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10074 Assert(pVmxTransient->cbInstr == 2);
10075 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10076 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10077 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10078 }
10079 else
10080 {
10081 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
10082 rc = VERR_EM_INTERPRETER;
10083 }
10084 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10085 return rc;
10086}
10087
10088
10089/**
10090 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10091 */
10092HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10093{
10094 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10095 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10096 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10097 AssertRCReturn(rc, rc);
10098
10099 PVM pVM = pVCpu->CTX_SUFF(pVM);
10100 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10101 if (RT_LIKELY(rc == VINF_SUCCESS))
10102 {
10103 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10104 Assert(pVmxTransient->cbInstr == 3);
10105 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10106 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10107 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10108 }
10109 else
10110 {
10111 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10112 rc = VERR_EM_INTERPRETER;
10113 }
10114 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10115 return rc;
10116}
10117
10118
10119/**
10120 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10121 */
10122HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10123{
10124 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10125 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10126 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10127 AssertRCReturn(rc, rc);
10128
10129 PVM pVM = pVCpu->CTX_SUFF(pVM);
10130 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10131 if (RT_LIKELY(rc == VINF_SUCCESS))
10132 {
10133 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10134 Assert(pVmxTransient->cbInstr == 2);
10135 }
10136 else
10137 {
10138 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10139 rc = VERR_EM_INTERPRETER;
10140 }
10141 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10142 return rc;
10143}
10144
10145
10146/**
10147 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10148 */
10149HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10150{
10151 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10152
10153 int rc = VERR_NOT_SUPPORTED;
10154 if (GIMAreHypercallsEnabled(pVCpu))
10155 {
10156 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10157 AssertRCReturn(rc, rc);
10158
10159 rc = GIMHypercall(pVCpu, pMixedCtx);
10160 }
10161 if (rc != VINF_SUCCESS)
10162 {
10163 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10164 rc = VINF_SUCCESS;
10165 }
10166
10167 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10168 return rc;
10169}
10170
10171
10172/**
10173 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10174 */
10175HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10176{
10177 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10178 PVM pVM = pVCpu->CTX_SUFF(pVM);
10179 Assert(!pVM->hm.s.fNestedPaging);
10180
10181 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10182 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10183 AssertRCReturn(rc, rc);
10184
10185 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10186 rc = VBOXSTRICTRC_VAL(rc2);
10187 if (RT_LIKELY(rc == VINF_SUCCESS))
10188 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10189 else
10190 {
10191 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10192 pVmxTransient->uExitQualification, rc));
10193 }
10194 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10195 return rc;
10196}
10197
10198
10199/**
10200 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10201 */
10202HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10203{
10204 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10205 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10206 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10207 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10208 AssertRCReturn(rc, rc);
10209
10210 PVM pVM = pVCpu->CTX_SUFF(pVM);
10211 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10212 if (RT_LIKELY(rc == VINF_SUCCESS))
10213 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10214 else
10215 {
10216 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10217 rc = VERR_EM_INTERPRETER;
10218 }
10219 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10220 return rc;
10221}
10222
10223
10224/**
10225 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10226 */
10227HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10228{
10229 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10230 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10231 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10232 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10233 AssertRCReturn(rc, rc);
10234
10235 PVM pVM = pVCpu->CTX_SUFF(pVM);
10236 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10237 rc = VBOXSTRICTRC_VAL(rc2);
10238 if (RT_LIKELY( rc == VINF_SUCCESS
10239 || rc == VINF_EM_HALT))
10240 {
10241 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10242 AssertRCReturn(rc3, rc3);
10243
10244 if ( rc == VINF_EM_HALT
10245 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10246 {
10247 rc = VINF_SUCCESS;
10248 }
10249 }
10250 else
10251 {
10252 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10253 rc = VERR_EM_INTERPRETER;
10254 }
10255 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10256 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10257 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10258 return rc;
10259}
10260
10261
10262/**
10263 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10264 */
10265HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10266{
10267 /*
10268 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10269 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10270 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10271 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10272 */
10273 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10274 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10275 HMVMX_RETURN_UNEXPECTED_EXIT();
10276}
10277
10278
10279/**
10280 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10281 */
10282HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10283{
10284 /*
10285 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10286 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10287 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10288 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10289 */
10290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10291 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10292 HMVMX_RETURN_UNEXPECTED_EXIT();
10293}
10294
10295
10296/**
10297 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10298 */
10299HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10300{
10301 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10302 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10303 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10304 HMVMX_RETURN_UNEXPECTED_EXIT();
10305}
10306
10307
10308/**
10309 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10310 */
10311HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10312{
10313 /*
10314 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10315 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10316 * See Intel spec. 25.3 "Other Causes of VM-exits".
10317 */
10318 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10319 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10320 HMVMX_RETURN_UNEXPECTED_EXIT();
10321}
10322
10323
10324/**
10325 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10326 * VM-exit.
10327 */
10328HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10329{
10330 /*
10331 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10332 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10333 *
10334 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10335 * See Intel spec. "23.8 Restrictions on VMX operation".
10336 */
10337 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10338 return VINF_SUCCESS;
10339}
10340
10341
10342/**
10343 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10344 * VM-exit.
10345 */
10346HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10347{
10348 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10349 return VINF_EM_RESET;
10350}
10351
10352
10353/**
10354 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10355 */
10356HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10357{
10358 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10359 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10360 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10361 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10362 AssertRCReturn(rc, rc);
10363
10364 pMixedCtx->rip++;
10365 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10366 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10367 rc = VINF_SUCCESS;
10368 else
10369 rc = VINF_EM_HALT;
10370
10371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10372 return rc;
10373}
10374
10375
10376/**
10377 * VM-exit handler for instructions that result in a #UD exception delivered to
10378 * the guest.
10379 */
10380HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10381{
10382 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10383 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10384 return VINF_SUCCESS;
10385}
10386
10387
10388/**
10389 * VM-exit handler for expiry of the VMX preemption timer.
10390 */
10391HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10392{
10393 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10394
10395 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10396 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10397
10398 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10399 PVM pVM = pVCpu->CTX_SUFF(pVM);
10400 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10401 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10402 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10403}
10404
10405
10406/**
10407 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10408 */
10409HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10410{
10411 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10412
10413 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10414 /** @todo check if XSETBV is supported by the recompiler. */
10415 return VERR_EM_INTERPRETER;
10416}
10417
10418
10419/**
10420 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10421 */
10422HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10423{
10424 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10425
10426 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10427 /** @todo implement EMInterpretInvpcid() */
10428 return VERR_EM_INTERPRETER;
10429}
10430
10431
10432/**
10433 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10434 * Error VM-exit.
10435 */
10436HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10437{
10438 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10439 AssertRCReturn(rc, rc);
10440
10441 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10442 AssertRCReturn(rc, rc);
10443
10444 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10445 NOREF(uInvalidReason);
10446
10447#ifdef VBOX_STRICT
10448 uint32_t uIntrState;
10449 HMVMXHCUINTREG uHCReg;
10450 uint64_t u64Val;
10451 uint32_t u32Val;
10452
10453 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10454 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10455 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10456 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10457 AssertRCReturn(rc, rc);
10458
10459 Log4(("uInvalidReason %u\n", uInvalidReason));
10460 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10461 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10462 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10463 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10464
10465 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10466 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10467 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10468 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10469 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10470 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10471 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10472 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10473 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10474 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10475 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10476 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10477#else
10478 NOREF(pVmxTransient);
10479#endif
10480
10481 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10482 return VERR_VMX_INVALID_GUEST_STATE;
10483}
10484
10485
10486/**
10487 * VM-exit handler for VM-entry failure due to an MSR-load
10488 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10489 */
10490HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10491{
10492 NOREF(pVmxTransient);
10493 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10494 HMVMX_RETURN_UNEXPECTED_EXIT();
10495}
10496
10497
10498/**
10499 * VM-exit handler for VM-entry failure due to a machine-check event
10500 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10501 */
10502HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10503{
10504 NOREF(pVmxTransient);
10505 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10506 HMVMX_RETURN_UNEXPECTED_EXIT();
10507}
10508
10509
10510/**
10511 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10512 * theory.
10513 */
10514HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10515{
10516 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10517 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10518 return VERR_VMX_UNDEFINED_EXIT_CODE;
10519}
10520
10521
10522/**
10523 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10524 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10525 * Conditional VM-exit.
10526 */
10527HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10528{
10529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10530
10531 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10532 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10533 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10534 return VERR_EM_INTERPRETER;
10535 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10536 HMVMX_RETURN_UNEXPECTED_EXIT();
10537}
10538
10539
10540/**
10541 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10542 */
10543HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10544{
10545 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10546
10547 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10549 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10550 return VERR_EM_INTERPRETER;
10551 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10552 HMVMX_RETURN_UNEXPECTED_EXIT();
10553}
10554
10555
10556/**
10557 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10558 */
10559HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10560{
10561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10562
10563 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10564 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10565 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10566 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10567 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10568 {
10569 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10570 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10571 }
10572 AssertRCReturn(rc, rc);
10573 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10574
10575#ifdef VBOX_STRICT
10576 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10577 {
10578 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10579 && pMixedCtx->ecx != MSR_K6_EFER)
10580 {
10581 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10582 HMVMX_RETURN_UNEXPECTED_EXIT();
10583 }
10584# if HC_ARCH_BITS == 64
10585 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10586 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10587 {
10588 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10589 HMVMX_RETURN_UNEXPECTED_EXIT();
10590 }
10591# endif
10592 }
10593#endif
10594
10595 PVM pVM = pVCpu->CTX_SUFF(pVM);
10596 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10597 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10598 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10599 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10600 if (RT_LIKELY(rc == VINF_SUCCESS))
10601 {
10602 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10603 Assert(pVmxTransient->cbInstr == 2);
10604 }
10605 return rc;
10606}
10607
10608
10609/**
10610 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10611 */
10612HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10613{
10614 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10615 PVM pVM = pVCpu->CTX_SUFF(pVM);
10616 int rc = VINF_SUCCESS;
10617
10618 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10619 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10620 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10621 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10622 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10623 {
10624 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10625 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10626 }
10627 AssertRCReturn(rc, rc);
10628 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10629
10630 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10631 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10632 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10633
10634 if (RT_LIKELY(rc == VINF_SUCCESS))
10635 {
10636 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10637
10638 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10639 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10640 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10641 {
10642 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10643 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10644 EMInterpretWrmsr() changes it. */
10645 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10646 }
10647 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10648 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10649 else if (pMixedCtx->ecx == MSR_K6_EFER)
10650 {
10651 /*
10652 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10653 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10654 * the other bits as well, SCE and NXE. See @bugref{7368}.
10655 */
10656 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10657 }
10658
10659 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10660 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10661 {
10662 switch (pMixedCtx->ecx)
10663 {
10664 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10665 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10666 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10667 case MSR_K8_FS_BASE: /* no break */
10668 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10669 case MSR_K6_EFER: /* already handled above */ break;
10670 default:
10671 {
10672 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10673 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10674#if HC_ARCH_BITS == 64
10675 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10676 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10677#endif
10678 break;
10679 }
10680 }
10681 }
10682#ifdef VBOX_STRICT
10683 else
10684 {
10685 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10686 switch (pMixedCtx->ecx)
10687 {
10688 case MSR_IA32_SYSENTER_CS:
10689 case MSR_IA32_SYSENTER_EIP:
10690 case MSR_IA32_SYSENTER_ESP:
10691 case MSR_K8_FS_BASE:
10692 case MSR_K8_GS_BASE:
10693 {
10694 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10695 HMVMX_RETURN_UNEXPECTED_EXIT();
10696 }
10697
10698 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10699 default:
10700 {
10701 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10702 {
10703 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10704 if (pMixedCtx->ecx != MSR_K6_EFER)
10705 {
10706 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10707 pMixedCtx->ecx));
10708 HMVMX_RETURN_UNEXPECTED_EXIT();
10709 }
10710 }
10711
10712#if HC_ARCH_BITS == 64
10713 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10714 {
10715 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10716 HMVMX_RETURN_UNEXPECTED_EXIT();
10717 }
10718#endif
10719 break;
10720 }
10721 }
10722 }
10723#endif /* VBOX_STRICT */
10724 }
10725 return rc;
10726}
10727
10728
10729/**
10730 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10731 */
10732HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10733{
10734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10735
10736 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10737 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10738 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10739 return VERR_EM_INTERPRETER;
10740 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10741 HMVMX_RETURN_UNEXPECTED_EXIT();
10742}
10743
10744
10745/**
10746 * VM-exit handler for when the TPR value is lowered below the specified
10747 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10748 */
10749HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10750{
10751 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10752 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10753
10754 /*
10755 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10756 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10757 * resume guest execution.
10758 */
10759 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10760 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10761 return VINF_SUCCESS;
10762}
10763
10764
10765/**
10766 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10767 * VM-exit.
10768 *
10769 * @retval VINF_SUCCESS when guest execution can continue.
10770 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10771 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10772 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10773 * recompiler.
10774 */
10775HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10776{
10777 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10778 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10779 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10780 AssertRCReturn(rc, rc);
10781
10782 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10783 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10784 PVM pVM = pVCpu->CTX_SUFF(pVM);
10785 switch (uAccessType)
10786 {
10787 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10788 {
10789#if 0
10790 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10791 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10792#else
10793 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10794 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10795 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10796#endif
10797 AssertRCReturn(rc, rc);
10798
10799 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10800 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10801 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10802 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10803
10804 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10805 {
10806 case 0: /* CR0 */
10807 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10808 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10809 break;
10810 case 2: /* CR2 */
10811 /* Nothing to do here, CR2 it's not part of the VMCS. */
10812 break;
10813 case 3: /* CR3 */
10814 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10815 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10816 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10817 break;
10818 case 4: /* CR4 */
10819 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10820 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10821 break;
10822 case 8: /* CR8 */
10823 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10824 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10825 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10826 break;
10827 default:
10828 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10829 break;
10830 }
10831
10832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10833 break;
10834 }
10835
10836 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10837 {
10838 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10839 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10840 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10841 AssertRCReturn(rc, rc);
10842 Assert( !pVM->hm.s.fNestedPaging
10843 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10844 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10845
10846 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10847 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10848 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10849
10850 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10851 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10852 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10853 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10854 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10855 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10856 break;
10857 }
10858
10859 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10860 {
10861 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10862 AssertRCReturn(rc, rc);
10863 rc = EMInterpretCLTS(pVM, pVCpu);
10864 AssertRCReturn(rc, rc);
10865 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10866 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10867 Log4(("CRX CLTS write rc=%d\n", rc));
10868 break;
10869 }
10870
10871 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10872 {
10873 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10874 AssertRCReturn(rc, rc);
10875 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10876 if (RT_LIKELY(rc == VINF_SUCCESS))
10877 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10878 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10879 Log4(("CRX LMSW write rc=%d\n", rc));
10880 break;
10881 }
10882
10883 default:
10884 {
10885 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10886 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10887 }
10888 }
10889
10890 /* Validate possible error codes. */
10891 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10892 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10893 if (RT_SUCCESS(rc))
10894 {
10895 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10896 AssertRCReturn(rc2, rc2);
10897 }
10898
10899 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10900 return rc;
10901}
10902
10903
10904/**
10905 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10906 * VM-exit.
10907 */
10908HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10909{
10910 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10911 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10912
10913 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10914 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10915 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10916 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10917 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10918 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10919 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10920 AssertRCReturn(rc2, rc2);
10921
10922 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10923 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10924 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10925 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10926 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10927 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10928 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10929 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10930
10931 /* I/O operation lookup arrays. */
10932 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10933 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10934
10935 VBOXSTRICTRC rcStrict;
10936 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10937 uint32_t const cbInstr = pVmxTransient->cbInstr;
10938 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10939 PVM pVM = pVCpu->CTX_SUFF(pVM);
10940 if (fIOString)
10941 {
10942#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10943 /*
10944 * INS/OUTS - I/O String instruction.
10945 *
10946 * Use instruction-information if available, otherwise fall back on
10947 * interpreting the instruction.
10948 */
10949 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10950 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10951 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10952 {
10953 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10954 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10955 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10956 AssertRCReturn(rc2, rc2);
10957 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10958 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10959 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10960 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10961 if (fIOWrite)
10962 {
10963 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10964 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10965 }
10966 else
10967 {
10968 /*
10969 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10970 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10971 * See Intel Instruction spec. for "INS".
10972 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10973 */
10974 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10975 }
10976 }
10977 else
10978 {
10979 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10980 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10981 AssertRCReturn(rc2, rc2);
10982 rcStrict = IEMExecOne(pVCpu);
10983 }
10984 /** @todo IEM needs to be setting these flags somehow. */
10985 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10986 fUpdateRipAlready = true;
10987#else
10988 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10989 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10990 if (RT_SUCCESS(rcStrict))
10991 {
10992 if (fIOWrite)
10993 {
10994 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10995 (DISCPUMODE)pDis->uAddrMode, cbValue);
10996 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10997 }
10998 else
10999 {
11000 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11001 (DISCPUMODE)pDis->uAddrMode, cbValue);
11002 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11003 }
11004 }
11005 else
11006 {
11007 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11008 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11009 }
11010#endif
11011 }
11012 else
11013 {
11014 /*
11015 * IN/OUT - I/O instruction.
11016 */
11017 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11018 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11019 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11020 if (fIOWrite)
11021 {
11022 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11023 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11024 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11025 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11026 }
11027 else
11028 {
11029 uint32_t u32Result = 0;
11030 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11031 if (IOM_SUCCESS(rcStrict))
11032 {
11033 /* Save result of I/O IN instr. in AL/AX/EAX. */
11034 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11035 }
11036 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11037 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11039 }
11040 }
11041
11042 if (IOM_SUCCESS(rcStrict))
11043 {
11044 if (!fUpdateRipAlready)
11045 {
11046 pMixedCtx->rip += cbInstr;
11047 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11048 }
11049
11050 /*
11051 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11052 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11053 */
11054 if (fIOString)
11055 {
11056 /** @todo Single-step for INS/OUTS with REP prefix? */
11057 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11058 }
11059 else if (fStepping)
11060 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11061
11062 /*
11063 * If any I/O breakpoints are armed, we need to check if one triggered
11064 * and take appropriate action.
11065 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11066 */
11067 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11068 AssertRCReturn(rc2, rc2);
11069
11070 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11071 * execution engines about whether hyper BPs and such are pending. */
11072 uint32_t const uDr7 = pMixedCtx->dr[7];
11073 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11074 && X86_DR7_ANY_RW_IO(uDr7)
11075 && (pMixedCtx->cr4 & X86_CR4_DE))
11076 || DBGFBpIsHwIoArmed(pVM)))
11077 {
11078 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11079
11080 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11081 VMMRZCallRing3Disable(pVCpu);
11082 HM_DISABLE_PREEMPT_IF_NEEDED();
11083
11084 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
11085
11086 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11087 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11088 {
11089 /* Raise #DB. */
11090 if (fIsGuestDbgActive)
11091 ASMSetDR6(pMixedCtx->dr[6]);
11092 if (pMixedCtx->dr[7] != uDr7)
11093 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11094
11095 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11096 }
11097 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11098 else if ( rcStrict2 != VINF_SUCCESS
11099 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11100 rcStrict = rcStrict2;
11101
11102 HM_RESTORE_PREEMPT_IF_NEEDED();
11103 VMMRZCallRing3Enable(pVCpu);
11104 }
11105 }
11106
11107#ifdef DEBUG
11108 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11109 Assert(!fIOWrite);
11110 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11111 Assert(fIOWrite);
11112 else
11113 {
11114 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11115 * statuses, that the VMM device and some others may return. See
11116 * IOM_SUCCESS() for guidance. */
11117 AssertMsg( RT_FAILURE(rcStrict)
11118 || rcStrict == VINF_SUCCESS
11119 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11120 || rcStrict == VINF_EM_DBG_BREAKPOINT
11121 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11122 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11123 }
11124#endif
11125
11126 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11127 return VBOXSTRICTRC_TODO(rcStrict);
11128}
11129
11130
11131/**
11132 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11133 * VM-exit.
11134 */
11135HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11136{
11137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11138
11139 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11140 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11141 AssertRCReturn(rc, rc);
11142 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11143 {
11144 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11145 AssertRCReturn(rc, rc);
11146 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11147 {
11148 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11149
11150 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11151 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11152
11153 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11154 Assert(!pVCpu->hm.s.Event.fPending);
11155 pVCpu->hm.s.Event.fPending = true;
11156 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11157 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11158 AssertRCReturn(rc, rc);
11159 if (fErrorCodeValid)
11160 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11161 else
11162 pVCpu->hm.s.Event.u32ErrCode = 0;
11163 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11164 && uVector == X86_XCPT_PF)
11165 {
11166 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11167 }
11168
11169 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11171 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11172 }
11173 }
11174
11175 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11176 * emulation. */
11177 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11178 return VERR_EM_INTERPRETER;
11179}
11180
11181
11182/**
11183 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11184 */
11185HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11186{
11187 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11188 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11189 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11190 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11191 AssertRCReturn(rc, rc);
11192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11193 return VINF_EM_DBG_STEPPED;
11194}
11195
11196
11197/**
11198 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11199 */
11200HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11201{
11202 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11203
11204 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11205 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11206 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11207 return VINF_SUCCESS;
11208 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11209 return rc;
11210
11211#if 0
11212 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11213 * just sync the whole thing. */
11214 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11215#else
11216 /* Aggressive state sync. for now. */
11217 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11218 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11219 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11220#endif
11221 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11222 AssertRCReturn(rc, rc);
11223
11224 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11225 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11226 switch (uAccessType)
11227 {
11228 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11229 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11230 {
11231 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11232 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11233 {
11234 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11235 }
11236
11237 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11238 GCPhys &= PAGE_BASE_GC_MASK;
11239 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11240 PVM pVM = pVCpu->CTX_SUFF(pVM);
11241 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11242 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11243
11244 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11245 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11246 CPUMCTX2CORE(pMixedCtx), GCPhys);
11247 rc = VBOXSTRICTRC_VAL(rc2);
11248 Log4(("ApicAccess rc=%d\n", rc));
11249 if ( rc == VINF_SUCCESS
11250 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11251 || rc == VERR_PAGE_NOT_PRESENT)
11252 {
11253 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11254 | HM_CHANGED_GUEST_RSP
11255 | HM_CHANGED_GUEST_RFLAGS
11256 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11257 rc = VINF_SUCCESS;
11258 }
11259 break;
11260 }
11261
11262 default:
11263 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11264 rc = VINF_EM_RAW_EMULATE_INSTR;
11265 break;
11266 }
11267
11268 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11269 return rc;
11270}
11271
11272
11273/**
11274 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11275 * VM-exit.
11276 */
11277HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11278{
11279 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11280
11281 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11282 if (pVmxTransient->fWasGuestDebugStateActive)
11283 {
11284 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11285 HMVMX_RETURN_UNEXPECTED_EXIT();
11286 }
11287
11288 int rc = VERR_INTERNAL_ERROR_5;
11289 if ( !DBGFIsStepping(pVCpu)
11290 && !pVCpu->hm.s.fSingleInstruction
11291 && !pVmxTransient->fWasHyperDebugStateActive)
11292 {
11293 /* Don't intercept MOV DRx and #DB any more. */
11294 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11295 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11296 AssertRCReturn(rc, rc);
11297
11298 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11299 {
11300#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11301 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11302 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11303 AssertRCReturn(rc, rc);
11304#endif
11305 }
11306
11307 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11308 VMMRZCallRing3Disable(pVCpu);
11309 HM_DISABLE_PREEMPT_IF_NEEDED();
11310
11311 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11312 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11313 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11314
11315 HM_RESTORE_PREEMPT_IF_NEEDED();
11316 VMMRZCallRing3Enable(pVCpu);
11317
11318#ifdef VBOX_WITH_STATISTICS
11319 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11320 AssertRCReturn(rc, rc);
11321 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11322 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11323 else
11324 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11325#endif
11326 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11327 return VINF_SUCCESS;
11328 }
11329
11330 /*
11331 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11332 * Update the segment registers and DR7 from the CPU.
11333 */
11334 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11335 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11336 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11337 AssertRCReturn(rc, rc);
11338 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11339
11340 PVM pVM = pVCpu->CTX_SUFF(pVM);
11341 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11342 {
11343 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11344 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11345 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11346 if (RT_SUCCESS(rc))
11347 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11348 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11349 }
11350 else
11351 {
11352 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11353 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11354 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11356 }
11357
11358 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11359 if (RT_SUCCESS(rc))
11360 {
11361 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11362 AssertRCReturn(rc2, rc2);
11363 }
11364 return rc;
11365}
11366
11367
11368/**
11369 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11370 * Conditional VM-exit.
11371 */
11372HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11373{
11374 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11375 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11376
11377 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11378 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11379 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11380 return VINF_SUCCESS;
11381 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11382 return rc;
11383
11384 RTGCPHYS GCPhys = 0;
11385 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11386
11387#if 0
11388 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11389#else
11390 /* Aggressive state sync. for now. */
11391 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11392 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11393 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11394#endif
11395 AssertRCReturn(rc, rc);
11396
11397 /*
11398 * If we succeed, resume guest execution.
11399 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11400 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11401 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11402 * weird case. See @bugref{6043}.
11403 */
11404 PVM pVM = pVCpu->CTX_SUFF(pVM);
11405 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11406 rc = VBOXSTRICTRC_VAL(rc2);
11407 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11408 if ( rc == VINF_SUCCESS
11409 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11410 || rc == VERR_PAGE_NOT_PRESENT)
11411 {
11412 /* Successfully handled MMIO operation. */
11413 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11414 | HM_CHANGED_GUEST_RSP
11415 | HM_CHANGED_GUEST_RFLAGS
11416 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11417 rc = VINF_SUCCESS;
11418 }
11419 return rc;
11420}
11421
11422
11423/**
11424 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11425 * VM-exit.
11426 */
11427HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11428{
11429 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11430 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11431
11432 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11433 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11434 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11435 return VINF_SUCCESS;
11436 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11437 return rc;
11438
11439 RTGCPHYS GCPhys = 0;
11440 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11441 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11442#if 0
11443 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11444#else
11445 /* Aggressive state sync. for now. */
11446 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11447 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11448 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11449#endif
11450 AssertRCReturn(rc, rc);
11451
11452 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11453 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11454
11455 RTGCUINT uErrorCode = 0;
11456 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11457 uErrorCode |= X86_TRAP_PF_ID;
11458 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11459 uErrorCode |= X86_TRAP_PF_RW;
11460 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11461 uErrorCode |= X86_TRAP_PF_P;
11462
11463 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11464
11465 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11466 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11467
11468 /* Handle the pagefault trap for the nested shadow table. */
11469 PVM pVM = pVCpu->CTX_SUFF(pVM);
11470 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11471 TRPMResetTrap(pVCpu);
11472
11473 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11474 if ( rc == VINF_SUCCESS
11475 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11476 || rc == VERR_PAGE_NOT_PRESENT)
11477 {
11478 /* Successfully synced our nested page tables. */
11479 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11480 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11481 | HM_CHANGED_GUEST_RSP
11482 | HM_CHANGED_GUEST_RFLAGS);
11483 return VINF_SUCCESS;
11484 }
11485
11486 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11487 return rc;
11488}
11489
11490/** @} */
11491
11492/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11493/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11494/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11495
11496/** @name VM-exit exception handlers.
11497 * @{
11498 */
11499
11500/**
11501 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11502 */
11503static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11504{
11505 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11507
11508 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11509 AssertRCReturn(rc, rc);
11510
11511 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11512 {
11513 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11514 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11515
11516 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11517 * provides VM-exit instruction length. If this causes problem later,
11518 * disassemble the instruction like it's done on AMD-V. */
11519 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11520 AssertRCReturn(rc2, rc2);
11521 return rc;
11522 }
11523
11524 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11525 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11526 return rc;
11527}
11528
11529
11530/**
11531 * VM-exit exception handler for #BP (Breakpoint exception).
11532 */
11533static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11534{
11535 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11536 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11537
11538 /** @todo Try optimize this by not saving the entire guest state unless
11539 * really needed. */
11540 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11541 AssertRCReturn(rc, rc);
11542
11543 PVM pVM = pVCpu->CTX_SUFF(pVM);
11544 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11545 if (rc == VINF_EM_RAW_GUEST_TRAP)
11546 {
11547 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11548 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11549 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11550 AssertRCReturn(rc, rc);
11551
11552 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11553 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11554 }
11555
11556 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11557 return rc;
11558}
11559
11560
11561/**
11562 * VM-exit exception handler for #DB (Debug exception).
11563 */
11564static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11565{
11566 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11568 Log6(("XcptDB\n"));
11569
11570 /*
11571 * Get the DR6-like values from the exit qualification and pass it to DBGF
11572 * for processing.
11573 */
11574 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11575 AssertRCReturn(rc, rc);
11576
11577 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11578 uint64_t uDR6 = X86_DR6_INIT_VAL;
11579 uDR6 |= ( pVmxTransient->uExitQualification
11580 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11581
11582 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11583 if (rc == VINF_EM_RAW_GUEST_TRAP)
11584 {
11585 /*
11586 * The exception was for the guest. Update DR6, DR7.GD and
11587 * IA32_DEBUGCTL.LBR before forwarding it.
11588 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11589 */
11590 VMMRZCallRing3Disable(pVCpu);
11591 HM_DISABLE_PREEMPT_IF_NEEDED();
11592
11593 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11594 pMixedCtx->dr[6] |= uDR6;
11595 if (CPUMIsGuestDebugStateActive(pVCpu))
11596 ASMSetDR6(pMixedCtx->dr[6]);
11597
11598 HM_RESTORE_PREEMPT_IF_NEEDED();
11599 VMMRZCallRing3Enable(pVCpu);
11600
11601 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11602 AssertRCReturn(rc, rc);
11603
11604 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11605 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11606
11607 /* Paranoia. */
11608 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11609 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11610
11611 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11612 AssertRCReturn(rc, rc);
11613
11614 /*
11615 * Raise #DB in the guest.
11616 *
11617 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11618 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11619 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11620 *
11621 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11622 */
11623 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11624 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11625 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11626 AssertRCReturn(rc, rc);
11627 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11628 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11629 return VINF_SUCCESS;
11630 }
11631
11632 /*
11633 * Not a guest trap, must be a hypervisor related debug event then.
11634 * Update DR6 in case someone is interested in it.
11635 */
11636 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11637 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11638 CPUMSetHyperDR6(pVCpu, uDR6);
11639
11640 return rc;
11641}
11642
11643
11644/**
11645 * VM-exit exception handler for #NM (Device-not-available exception: floating
11646 * point exception).
11647 */
11648static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11649{
11650 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11651
11652 /* We require CR0 and EFER. EFER is always up-to-date. */
11653 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11654 AssertRCReturn(rc, rc);
11655
11656 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11657 VMMRZCallRing3Disable(pVCpu);
11658 HM_DISABLE_PREEMPT_IF_NEEDED();
11659
11660 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11661 if (pVmxTransient->fWasGuestFPUStateActive)
11662 {
11663 rc = VINF_EM_RAW_GUEST_TRAP;
11664 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11665 }
11666 else
11667 {
11668#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11669 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11670#endif
11671 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11672 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11673 }
11674
11675 HM_RESTORE_PREEMPT_IF_NEEDED();
11676 VMMRZCallRing3Enable(pVCpu);
11677
11678 if (rc == VINF_SUCCESS)
11679 {
11680 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11681 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11683 pVCpu->hm.s.fUseGuestFpu = true;
11684 }
11685 else
11686 {
11687 /* Forward #NM to the guest. */
11688 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11689 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11690 AssertRCReturn(rc, rc);
11691 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11692 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11694 }
11695
11696 return VINF_SUCCESS;
11697}
11698
11699
11700/**
11701 * VM-exit exception handler for #GP (General-protection exception).
11702 *
11703 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11704 */
11705static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11706{
11707 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11708 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11709
11710 int rc = VERR_INTERNAL_ERROR_5;
11711 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11712 {
11713#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11714 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11715 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11716 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11717 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11718 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11719 AssertRCReturn(rc, rc);
11720 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11721 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11722 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11723 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11724 return rc;
11725#else
11726 /* We don't intercept #GP. */
11727 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11728 NOREF(pVmxTransient);
11729 return VERR_VMX_UNEXPECTED_EXCEPTION;
11730#endif
11731 }
11732
11733 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11734 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11735
11736 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11737 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11738 AssertRCReturn(rc, rc);
11739
11740 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11741 uint32_t cbOp = 0;
11742 PVM pVM = pVCpu->CTX_SUFF(pVM);
11743 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11744 if (RT_SUCCESS(rc))
11745 {
11746 rc = VINF_SUCCESS;
11747 Assert(cbOp == pDis->cbInstr);
11748 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11749 switch (pDis->pCurInstr->uOpcode)
11750 {
11751 case OP_CLI:
11752 {
11753 pMixedCtx->eflags.Bits.u1IF = 0;
11754 pMixedCtx->rip += pDis->cbInstr;
11755 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11756 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11757 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11758 break;
11759 }
11760
11761 case OP_STI:
11762 {
11763 pMixedCtx->eflags.Bits.u1IF = 1;
11764 pMixedCtx->rip += pDis->cbInstr;
11765 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11766 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11767 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11768 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11770 break;
11771 }
11772
11773 case OP_HLT:
11774 {
11775 rc = VINF_EM_HALT;
11776 pMixedCtx->rip += pDis->cbInstr;
11777 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11778 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11779 break;
11780 }
11781
11782 case OP_POPF:
11783 {
11784 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11785 uint32_t cbParm;
11786 uint32_t uMask;
11787 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11788 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11789 {
11790 cbParm = 4;
11791 uMask = 0xffffffff;
11792 }
11793 else
11794 {
11795 cbParm = 2;
11796 uMask = 0xffff;
11797 }
11798
11799 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11800 RTGCPTR GCPtrStack = 0;
11801 X86EFLAGS Eflags;
11802 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11803 &GCPtrStack);
11804 if (RT_SUCCESS(rc))
11805 {
11806 Assert(sizeof(Eflags.u32) >= cbParm);
11807 Eflags.u32 = 0;
11808 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11809 }
11810 if (RT_FAILURE(rc))
11811 {
11812 rc = VERR_EM_INTERPRETER;
11813 break;
11814 }
11815 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11816 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11817 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11818 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11819 pMixedCtx->esp += cbParm;
11820 pMixedCtx->esp &= uMask;
11821 pMixedCtx->rip += pDis->cbInstr;
11822 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11823 | HM_CHANGED_GUEST_RSP
11824 | HM_CHANGED_GUEST_RFLAGS);
11825 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11826 if (fStepping)
11827 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11828
11829 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11830 break;
11831 }
11832
11833 case OP_PUSHF:
11834 {
11835 uint32_t cbParm;
11836 uint32_t uMask;
11837 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11838 {
11839 cbParm = 4;
11840 uMask = 0xffffffff;
11841 }
11842 else
11843 {
11844 cbParm = 2;
11845 uMask = 0xffff;
11846 }
11847
11848 /* Get the stack pointer & push the contents of eflags onto the stack. */
11849 RTGCPTR GCPtrStack = 0;
11850 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11851 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11852 if (RT_FAILURE(rc))
11853 {
11854 rc = VERR_EM_INTERPRETER;
11855 break;
11856 }
11857 X86EFLAGS Eflags = pMixedCtx->eflags;
11858 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11859 Eflags.Bits.u1RF = 0;
11860 Eflags.Bits.u1VM = 0;
11861
11862 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11863 if (RT_FAILURE(rc))
11864 {
11865 rc = VERR_EM_INTERPRETER;
11866 break;
11867 }
11868 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11869 pMixedCtx->esp -= cbParm;
11870 pMixedCtx->esp &= uMask;
11871 pMixedCtx->rip += pDis->cbInstr;
11872 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11873 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11874 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11875 break;
11876 }
11877
11878 case OP_IRET:
11879 {
11880 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11881 * instruction reference. */
11882 RTGCPTR GCPtrStack = 0;
11883 uint32_t uMask = 0xffff;
11884 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11885 uint16_t aIretFrame[3];
11886 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11887 {
11888 rc = VERR_EM_INTERPRETER;
11889 break;
11890 }
11891 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11892 &GCPtrStack);
11893 if (RT_SUCCESS(rc))
11894 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11895 if (RT_FAILURE(rc))
11896 {
11897 rc = VERR_EM_INTERPRETER;
11898 break;
11899 }
11900 pMixedCtx->eip = 0;
11901 pMixedCtx->ip = aIretFrame[0];
11902 pMixedCtx->cs.Sel = aIretFrame[1];
11903 pMixedCtx->cs.ValidSel = aIretFrame[1];
11904 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11905 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11906 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11907 pMixedCtx->sp += sizeof(aIretFrame);
11908 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11909 | HM_CHANGED_GUEST_SEGMENT_REGS
11910 | HM_CHANGED_GUEST_RSP
11911 | HM_CHANGED_GUEST_RFLAGS);
11912 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11913 if (fStepping)
11914 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11915 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11917 break;
11918 }
11919
11920 case OP_INT:
11921 {
11922 uint16_t uVector = pDis->Param1.uValue & 0xff;
11923 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11924 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11925 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11926 break;
11927 }
11928
11929 case OP_INTO:
11930 {
11931 if (pMixedCtx->eflags.Bits.u1OF)
11932 {
11933 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11934 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11935 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11936 }
11937 break;
11938 }
11939
11940 default:
11941 {
11942 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11943 EMCODETYPE_SUPERVISOR);
11944 rc = VBOXSTRICTRC_VAL(rc2);
11945 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11946 /** @todo We have to set pending-debug exceptions here when the guest is
11947 * single-stepping depending on the instruction that was interpreted. */
11948 Log4(("#GP rc=%Rrc\n", rc));
11949 break;
11950 }
11951 }
11952 }
11953 else
11954 rc = VERR_EM_INTERPRETER;
11955
11956 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11957 ("#GP Unexpected rc=%Rrc\n", rc));
11958 return rc;
11959}
11960
11961
11962#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11963/**
11964 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11965 * the exception reported in the VMX transient structure back into the VM.
11966 *
11967 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11968 * up-to-date.
11969 */
11970static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11971{
11972 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11973
11974 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11975 hmR0VmxCheckExitDueToEventDelivery(). */
11976 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11977 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11978 AssertRCReturn(rc, rc);
11979 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11980
11981#ifdef DEBUG_ramshankar
11982 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11983 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11984 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11985#endif
11986
11987 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11988 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11989 return VINF_SUCCESS;
11990}
11991#endif
11992
11993
11994/**
11995 * VM-exit exception handler for #PF (Page-fault exception).
11996 */
11997static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11998{
11999 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12000 PVM pVM = pVCpu->CTX_SUFF(pVM);
12001 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12002 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12003 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12004 AssertRCReturn(rc, rc);
12005
12006#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12007 if (pVM->hm.s.fNestedPaging)
12008 {
12009 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12010 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
12011 {
12012 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12013 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12014 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12015 }
12016 else
12017 {
12018 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12019 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12020 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12021 }
12022 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12023 return rc;
12024 }
12025#else
12026 Assert(!pVM->hm.s.fNestedPaging);
12027 NOREF(pVM);
12028#endif
12029
12030 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12031 AssertRCReturn(rc, rc);
12032
12033 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12034 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12035
12036 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12037 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12038 (RTGCPTR)pVmxTransient->uExitQualification);
12039
12040 Log4(("#PF: rc=%Rrc\n", rc));
12041 if (rc == VINF_SUCCESS)
12042 {
12043 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12044 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12045 * memory? We don't update the whole state here... */
12046 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12047 | HM_CHANGED_GUEST_RSP
12048 | HM_CHANGED_GUEST_RFLAGS
12049 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12050 TRPMResetTrap(pVCpu);
12051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12052 return rc;
12053 }
12054 else if (rc == VINF_EM_RAW_GUEST_TRAP)
12055 {
12056 if (!pVmxTransient->fVectoringPF)
12057 {
12058 /* It's a guest page fault and needs to be reflected to the guest. */
12059 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12060 TRPMResetTrap(pVCpu);
12061 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12062 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12063 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12064 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12065 }
12066 else
12067 {
12068 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12069 TRPMResetTrap(pVCpu);
12070 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12071 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12072 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12073 }
12074
12075 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12076 return VINF_SUCCESS;
12077 }
12078
12079 TRPMResetTrap(pVCpu);
12080 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12081 return rc;
12082}
12083
12084/** @} */
12085
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette