VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49288

Last change on this file since 49288 was 49288, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Fix for MOV DRx intercepts on 32-bit hosts with 64-bit guests.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 461.3 KB
Line 
1/* $Id: HMVMXR0.cpp 49288 2013-10-25 13:11:56Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#define HMVMX_ALWAYS_FLUSH_TLB
46#endif
47
48
49/*******************************************************************************
50* Defined Constants And Macros *
51*******************************************************************************/
52#if defined(RT_ARCH_AMD64)
53# define HMVMX_IS_64BIT_HOST_MODE() (true)
54typedef RTHCUINTREG HMVMXHCUINTREG;
55#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56extern "C" uint32_t g_fVMXIs64bitHost;
57# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58typedef uint64_t HMVMXHCUINTREG;
59#else
60# define HMVMX_IS_64BIT_HOST_MODE() (false)
61typedef RTHCUINTREG HMVMXHCUINTREG;
62#endif
63
64/** Use the function table. */
65#define HMVMX_USE_FUNCTION_TABLE
66
67/**
68 * The maximum number of MSRs we are willing to swap during a world-switch.
69 * Intel claims 512/check capability MSR, we don't want to do anywhere close
70 * to that. See Intel spec. 24.7.2 "VM-Exit Controls for MSRs"
71 *
72 * Bump this count as and when required, there's no backward compatibility
73 * requirement.
74 */
75#define HMVMX_MAX_SWAP_MSR_COUNT 5
76
77/** Determine which tagged-TLB flush handler to use. */
78#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
79#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
80#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
81#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
82
83/** @name Updated-guest-state flags.
84 * @{ */
85#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
86#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
87#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
88#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
89#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
90#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
91#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
92#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
93#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
94#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
95#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
96#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
97#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
98#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
99#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
100#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
101#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
102#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
103#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
104#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
105#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
106 | HMVMX_UPDATED_GUEST_RSP \
107 | HMVMX_UPDATED_GUEST_RFLAGS \
108 | HMVMX_UPDATED_GUEST_CR0 \
109 | HMVMX_UPDATED_GUEST_CR3 \
110 | HMVMX_UPDATED_GUEST_CR4 \
111 | HMVMX_UPDATED_GUEST_GDTR \
112 | HMVMX_UPDATED_GUEST_IDTR \
113 | HMVMX_UPDATED_GUEST_LDTR \
114 | HMVMX_UPDATED_GUEST_TR \
115 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
116 | HMVMX_UPDATED_GUEST_DEBUG \
117 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
118 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
119 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
120 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
121 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
122 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
123 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
124 | HMVMX_UPDATED_GUEST_APIC_STATE)
125/** @} */
126
127/** @name
128 * Flags to skip redundant reads of some common VMCS fields that are not part of
129 * the guest-CPU state but are in the transient structure.
130 */
131#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
132#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
133#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
134#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
135#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
136#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
137#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
138/** @} */
139
140/** @name
141 * States of the VMCS.
142 *
143 * This does not reflect all possible VMCS states but currently only those
144 * needed for maintaining the VMCS consistently even when thread-context hooks
145 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
146 */
147#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
148#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
149#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
150/** @} */
151
152/**
153 * Exception bitmap mask for real-mode guests (real-on-v86).
154 *
155 * We need to intercept all exceptions manually (except #PF). #NM is also
156 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
157 * even in real-mode if we have Nested Paging support.
158 */
159#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
160 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
161 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
162 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
163 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
164 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
165 | RT_BIT(X86_XCPT_XF))
166
167/**
168 * Exception bitmap mask for all contributory exceptions.
169 *
170 * Page fault is deliberately excluded here as it's conditional as to whether
171 * it's contributory or benign. Page faults are handled separately.
172 */
173#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
174 | RT_BIT(X86_XCPT_DE))
175
176/** Maximum VM-instruction error number. */
177#define HMVMX_INSTR_ERROR_MAX 28
178
179/** Profiling macro. */
180#ifdef HM_PROFILE_EXIT_DISPATCH
181# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
183#else
184# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
185# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
186#endif
187
188/** Assert that preemption is disabled or covered by thread-context hooks. */
189#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
191
192/** Assert that we haven't migrated CPUs when thread-context hooks are not
193 * used. */
194#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
195 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
196 ("Illegal migration! Entered on CPU %u Current %u\n", \
197 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
198
199/** Helper macro for VM-exit handlers called unexpectedly. */
200#define HMVMX_RETURN_UNEXPECTED_EXIT() \
201 do { \
202 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
203 return VERR_VMX_UNEXPECTED_EXIT; \
204 } while (0)
205
206
207/*******************************************************************************
208* Structures and Typedefs *
209*******************************************************************************/
210/**
211 * VMX transient state.
212 *
213 * A state structure for holding miscellaneous information across
214 * VMX non-root operation and restored after the transition.
215 */
216typedef struct VMXTRANSIENT
217{
218 /** The host's rflags/eflags. */
219 RTCCUINTREG uEflags;
220#if HC_ARCH_BITS == 32
221 uint32_t u32Alignment0;
222#endif
223 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
224 uint64_t u64LStarMsr;
225 /** The guest's TPR value used for TPR shadowing. */
226 uint8_t u8GuestTpr;
227 /** Alignment. */
228 uint8_t abAlignment0[7];
229
230 /** The basic VM-exit reason. */
231 uint16_t uExitReason;
232 /** Alignment. */
233 uint16_t u16Alignment0;
234 /** The VM-exit interruption error code. */
235 uint32_t uExitIntErrorCode;
236 /** The VM-exit exit qualification. */
237 uint64_t uExitQualification;
238
239 /** The VM-exit interruption-information field. */
240 uint32_t uExitIntInfo;
241 /** The VM-exit instruction-length field. */
242 uint32_t cbInstr;
243 /** The VM-exit instruction-information field. */
244 union
245 {
246 /** Plain unsigned int representation. */
247 uint32_t u;
248 /** INS and OUTS information. */
249 struct
250 {
251 uint32_t u6Reserved0 : 7;
252 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253 uint32_t u3AddrSize : 3;
254 uint32_t u5Reserved1 : 5;
255 /** The segment register (X86_SREG_XXX). */
256 uint32_t iSegReg : 3;
257 uint32_t uReserved2 : 14;
258 } StrIo;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest FPU was active at the time of VM-exit. */
281 bool fWasGuestFPUStateActive;
282 /** Whether the guest debug state was active at the time of VM-exit. */
283 bool fWasGuestDebugStateActive;
284 /** Whether the hyper debug state was active at the time of VM-exit. */
285 bool fWasHyperDebugStateActive;
286 /** Whether TSC-offsetting should be setup before VM-entry. */
287 bool fUpdateTscOffsettingAndPreemptTimer;
288 /** Whether the VM-exit was caused by a page-fault during delivery of a
289 * contributory exception or a page-fault. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322
323/**
324 * VMX VM-exit handler.
325 *
326 * @returns VBox status code.
327 * @param pVCpu Pointer to the VMCPU.
328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
329 * out-of-sync. Make sure to update the required
330 * fields before using them.
331 * @param pVmxTransient Pointer to the VMX-transient structure.
332 */
333#ifndef HMVMX_USE_FUNCTION_TABLE
334typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335#else
336typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
337/** Pointer to VM-exit handler. */
338typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
339#endif
340
341
342/*******************************************************************************
343* Internal Functions *
344*******************************************************************************/
345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
346static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
347static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
348static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
350#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
351static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
352#endif
353#ifndef HMVMX_USE_FUNCTION_TABLE
354DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
355# define HMVMX_EXIT_DECL static int
356#else
357# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
358#endif
359
360/** @name VM-exit handlers.
361 * @{
362 */
363static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
364static FNVMXEXITHANDLER hmR0VmxExitExtInt;
365static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
366static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
367static FNVMXEXITHANDLER hmR0VmxExitSipi;
368static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
369static FNVMXEXITHANDLER hmR0VmxExitSmi;
370static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
371static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
372static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
373static FNVMXEXITHANDLER hmR0VmxExitCpuid;
374static FNVMXEXITHANDLER hmR0VmxExitGetsec;
375static FNVMXEXITHANDLER hmR0VmxExitHlt;
376static FNVMXEXITHANDLER hmR0VmxExitInvd;
377static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
378static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
379static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
380static FNVMXEXITHANDLER hmR0VmxExitRsm;
381static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
382static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
383static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
384static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
385static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
386static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
387static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
388static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
389static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
390static FNVMXEXITHANDLER hmR0VmxExitMwait;
391static FNVMXEXITHANDLER hmR0VmxExitMtf;
392static FNVMXEXITHANDLER hmR0VmxExitMonitor;
393static FNVMXEXITHANDLER hmR0VmxExitPause;
394static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
395static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
396static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
397static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
398static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
404static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407/** @} */
408
409static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
417
418/*******************************************************************************
419* Global Variables *
420*******************************************************************************/
421#ifdef HMVMX_USE_FUNCTION_TABLE
422
423/**
424 * VMX_EXIT dispatch table.
425 */
426static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
427{
428 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
429 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
430 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
431 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
432 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
433 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
434 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
435 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
436 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
437 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
438 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
439 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
440 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
441 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
442 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
443 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
444 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
445 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
446 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
447 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
448 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
449 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
450 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
451 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
452 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
453 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
454 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
455 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
456 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
457 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
458 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
459 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
460 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
461 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
462 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
463 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
464 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
465 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
466 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
468 /* 40 UNDEFINED */ hmR0VmxExitPause,
469 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
470 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
471 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
472 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
473 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
474 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
475 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
476 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
477 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
478 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
479 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
480 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
481 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
482 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
483 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
484 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
485 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
486 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
487 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
488};
489#endif /* HMVMX_USE_FUNCTION_TABLE */
490
491#ifdef VBOX_STRICT
492static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
493{
494 /* 0 */ "(Not Used)",
495 /* 1 */ "VMCALL executed in VMX root operation.",
496 /* 2 */ "VMCLEAR with invalid physical address.",
497 /* 3 */ "VMCLEAR with VMXON pointer.",
498 /* 4 */ "VMLAUNCH with non-clear VMCS.",
499 /* 5 */ "VMRESUME with non-launched VMCS.",
500 /* 6 */ "VMRESUME after VMXOFF",
501 /* 7 */ "VM entry with invalid control fields.",
502 /* 8 */ "VM entry with invalid host state fields.",
503 /* 9 */ "VMPTRLD with invalid physical address.",
504 /* 10 */ "VMPTRLD with VMXON pointer.",
505 /* 11 */ "VMPTRLD with incorrect revision identifier.",
506 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
507 /* 13 */ "VMWRITE to read-only VMCS component.",
508 /* 14 */ "(Not Used)",
509 /* 15 */ "VMXON executed in VMX root operation.",
510 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
511 /* 17 */ "VM entry with non-launched executing VMCS.",
512 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
513 /* 19 */ "VMCALL with non-clear VMCS.",
514 /* 20 */ "VMCALL with invalid VM-exit control fields.",
515 /* 21 */ "(Not Used)",
516 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
517 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
518 /* 24 */ "VMCALL with invalid SMM-monitor features.",
519 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
520 /* 26 */ "VM entry with events blocked by MOV SS.",
521 /* 27 */ "(Not Used)",
522 /* 28 */ "Invalid operand to INVEPT/INVVPID."
523};
524#endif /* VBOX_STRICT */
525
526
527
528/**
529 * Updates the VM's last error record. If there was a VMX instruction error,
530 * reads the error data from the VMCS and updates VCPU's last error record as
531 * well.
532 *
533 * @param pVM Pointer to the VM.
534 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
535 * VERR_VMX_UNABLE_TO_START_VM or
536 * VERR_VMX_INVALID_VMCS_FIELD).
537 * @param rc The error code.
538 */
539static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
540{
541 AssertPtr(pVM);
542 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
543 || rc == VERR_VMX_UNABLE_TO_START_VM)
544 {
545 AssertPtrReturnVoid(pVCpu);
546 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
547 }
548 pVM->hm.s.lLastError = rc;
549}
550
551
552/**
553 * Reads the VM-entry interruption-information field from the VMCS into the VMX
554 * transient structure.
555 *
556 * @returns VBox status code.
557 * @param pVmxTransient Pointer to the VMX transient structure.
558 *
559 * @remarks No-long-jump zone!!!
560 */
561DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
562{
563 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
564 AssertRCReturn(rc, rc);
565 return VINF_SUCCESS;
566}
567
568
569/**
570 * Reads the VM-entry exception error code field from the VMCS into
571 * the VMX transient structure.
572 *
573 * @returns VBox status code.
574 * @param pVmxTransient Pointer to the VMX transient structure.
575 *
576 * @remarks No-long-jump zone!!!
577 */
578DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
579{
580 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
581 AssertRCReturn(rc, rc);
582 return VINF_SUCCESS;
583}
584
585
586/**
587 * Reads the VM-entry exception error code field from the VMCS into
588 * the VMX transient structure.
589 *
590 * @returns VBox status code.
591 * @param pVCpu Pointer to the VMCPU.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-exit interruption-information field from the VMCS into the VMX
606 * transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVCpu Pointer to the VMCPU.
610 * @param pVmxTransient Pointer to the VMX transient structure.
611 */
612DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
613{
614 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
615 {
616 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
617 AssertRCReturn(rc, rc);
618 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
619 }
620 return VINF_SUCCESS;
621}
622
623
624/**
625 * Reads the VM-exit interruption error code from the VMCS into the VMX
626 * transient structure.
627 *
628 * @returns VBox status code.
629 * @param pVCpu Pointer to the VMCPU.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 */
632DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
633{
634 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
635 {
636 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
637 AssertRCReturn(rc, rc);
638 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
639 }
640 return VINF_SUCCESS;
641}
642
643
644/**
645 * Reads the VM-exit instruction length field from the VMCS into the VMX
646 * transient structure.
647 *
648 * @returns VBox status code.
649 * @param pVCpu Pointer to the VMCPU.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
657 AssertRCReturn(rc, rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit instruction-information field from the VMCS into
666 * the VMX transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVCpu The cross context per CPU structure.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit qualification from the VMCS into the VMX transient structure.
686 *
687 * @returns VBox status code.
688 * @param pVCpu Pointer to the VMCPU.
689 * @param pVmxTransient Pointer to the VMX transient structure.
690 */
691DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
692{
693 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
694 {
695 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
696 AssertRCReturn(rc, rc);
697 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
698 }
699 return VINF_SUCCESS;
700}
701
702
703/**
704 * Reads the IDT-vectoring information field from the VMCS into the VMX
705 * transient structure.
706 *
707 * @returns VBox status code.
708 * @param pVmxTransient Pointer to the VMX transient structure.
709 *
710 * @remarks No-long-jump zone!!!
711 */
712DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
713{
714 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
715 {
716 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
717 AssertRCReturn(rc, rc);
718 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
719 }
720 return VINF_SUCCESS;
721}
722
723
724/**
725 * Reads the IDT-vectoring error code from the VMCS into the VMX
726 * transient structure.
727 *
728 * @returns VBox status code.
729 * @param pVmxTransient Pointer to the VMX transient structure.
730 */
731DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
732{
733 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
734 {
735 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
736 AssertRCReturn(rc, rc);
737 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
738 }
739 return VINF_SUCCESS;
740}
741
742
743/**
744 * Enters VMX root mode operation on the current CPU.
745 *
746 * @returns VBox status code.
747 * @param pVM Pointer to the VM (optional, can be NULL, after
748 * a resume).
749 * @param HCPhysCpuPage Physical address of the VMXON region.
750 * @param pvCpuPage Pointer to the VMXON region.
751 */
752static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
753{
754 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
755 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 if (pVM)
759 {
760 /* Write the VMCS revision dword to the VMXON region. */
761 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
762 }
763
764 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
765 RTCCUINTREG uEflags = ASMIntDisableFlags();
766
767 /* Enable the VMX bit in CR4 if necessary. */
768 RTCCUINTREG uCr4 = ASMGetCR4();
769 if (!(uCr4 & X86_CR4_VMXE))
770 ASMSetCR4(uCr4 | X86_CR4_VMXE);
771
772 /* Enter VMX root mode. */
773 int rc = VMXEnable(HCPhysCpuPage);
774 if (RT_FAILURE(rc))
775 ASMSetCR4(uCr4);
776
777 /* Restore interrupts. */
778 ASMSetFlags(uEflags);
779 return rc;
780}
781
782
783/**
784 * Exits VMX root mode operation on the current CPU.
785 *
786 * @returns VBox status code.
787 */
788static int hmR0VmxLeaveRootMode(void)
789{
790 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
791
792 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
793 RTCCUINTREG uEflags = ASMIntDisableFlags();
794
795 /* If we're for some reason not in VMX root mode, then don't leave it. */
796 RTCCUINTREG uHostCR4 = ASMGetCR4();
797
798 int rc;
799 if (uHostCR4 & X86_CR4_VMXE)
800 {
801 /* Exit VMX root mode and clear the VMX bit in CR4. */
802 VMXDisable();
803 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
804 rc = VINF_SUCCESS;
805 }
806 else
807 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
808
809 /* Restore interrupts. */
810 ASMSetFlags(uEflags);
811 return rc;
812}
813
814
815/**
816 * Allocates and maps one physically contiguous page. The allocated page is
817 * zero'd out. (Used by various VT-x structures).
818 *
819 * @returns IPRT status code.
820 * @param pMemObj Pointer to the ring-0 memory object.
821 * @param ppVirt Where to store the virtual address of the
822 * allocation.
823 * @param pPhys Where to store the physical address of the
824 * allocation.
825 */
826DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
827{
828 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
829 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
830 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
831
832 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
833 if (RT_FAILURE(rc))
834 return rc;
835 *ppVirt = RTR0MemObjAddress(*pMemObj);
836 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
837 ASMMemZero32(*ppVirt, PAGE_SIZE);
838 return VINF_SUCCESS;
839}
840
841
842/**
843 * Frees and unmaps an allocated physical page.
844 *
845 * @param pMemObj Pointer to the ring-0 memory object.
846 * @param ppVirt Where to re-initialize the virtual address of
847 * allocation as 0.
848 * @param pHCPhys Where to re-initialize the physical address of the
849 * allocation as 0.
850 */
851DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
852{
853 AssertPtr(pMemObj);
854 AssertPtr(ppVirt);
855 AssertPtr(pHCPhys);
856 if (*pMemObj != NIL_RTR0MEMOBJ)
857 {
858 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
859 AssertRC(rc);
860 *pMemObj = NIL_RTR0MEMOBJ;
861 *ppVirt = 0;
862 *pHCPhys = 0;
863 }
864}
865
866
867/**
868 * Worker function to free VT-x related structures.
869 *
870 * @returns IPRT status code.
871 * @param pVM Pointer to the VM.
872 */
873static void hmR0VmxStructsFree(PVM pVM)
874{
875 for (VMCPUID i = 0; i < pVM->cCpus; i++)
876 {
877 PVMCPU pVCpu = &pVM->aCpus[i];
878 AssertPtr(pVCpu);
879
880#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
881 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
883#endif
884
885 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
887
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
890 }
891
892 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
893#ifdef VBOX_WITH_CRASHDUMP_MAGIC
894 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
895#endif
896}
897
898
899/**
900 * Worker function to allocate VT-x related VM structures.
901 *
902 * @returns IPRT status code.
903 * @param pVM Pointer to the VM.
904 */
905static int hmR0VmxStructsAlloc(PVM pVM)
906{
907 /*
908 * Initialize members up-front so we can cleanup properly on allocation failure.
909 */
910#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
911 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
912 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
913 pVM->hm.s.vmx.HCPhys##a_Name = 0;
914
915#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
919
920#ifdef VBOX_WITH_CRASHDUMP_MAGIC
921 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
922#endif
923 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
924
925 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
926 for (VMCPUID i = 0; i < pVM->cCpus; i++)
927 {
928 PVMCPU pVCpu = &pVM->aCpus[i];
929 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
931 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
932#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935#endif
936 }
937#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
938#undef VMXLOCAL_INIT_VM_MEMOBJ
939
940 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
941 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
942 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
943 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
944
945 /*
946 * Allocate all the VT-x structures.
947 */
948 int rc = VINF_SUCCESS;
949#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951 if (RT_FAILURE(rc))
952 goto cleanup;
953 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
954 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
955#endif
956
957 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
958 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
959 {
960 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
961 &pVM->hm.s.vmx.HCPhysApicAccess);
962 if (RT_FAILURE(rc))
963 goto cleanup;
964 }
965
966 /*
967 * Initialize per-VCPU VT-x structures.
968 */
969 for (VMCPUID i = 0; i < pVM->cCpus; i++)
970 {
971 PVMCPU pVCpu = &pVM->aCpus[i];
972 AssertPtr(pVCpu);
973
974 /* Allocate the VM control structure (VMCS). */
975 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978
979 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
980 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
981 {
982 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
983 &pVCpu->hm.s.vmx.HCPhysVirtApic);
984 if (RT_FAILURE(rc))
985 goto cleanup;
986 }
987
988 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
990 {
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
992 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
996 }
997
998#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
999 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003
1004 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1005 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008#endif
1009 }
1010
1011 return VINF_SUCCESS;
1012
1013cleanup:
1014 hmR0VmxStructsFree(pVM);
1015 return rc;
1016}
1017
1018
1019/**
1020 * Does global VT-x initialization (called during module initialization).
1021 *
1022 * @returns VBox status code.
1023 */
1024VMMR0DECL(int) VMXR0GlobalInit(void)
1025{
1026#ifdef HMVMX_USE_FUNCTION_TABLE
1027 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1028# ifdef VBOX_STRICT
1029 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1030 Assert(g_apfnVMExitHandlers[i]);
1031# endif
1032#endif
1033 return VINF_SUCCESS;
1034}
1035
1036
1037/**
1038 * Does global VT-x termination (called during module termination).
1039 */
1040VMMR0DECL(void) VMXR0GlobalTerm()
1041{
1042 /* Nothing to do currently. */
1043}
1044
1045
1046/**
1047 * Sets up and activates VT-x on the current CPU.
1048 *
1049 * @returns VBox status code.
1050 * @param pCpu Pointer to the global CPU info struct.
1051 * @param pVM Pointer to the VM (can be NULL after a host resume
1052 * operation).
1053 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1054 * fEnabledByHost is true).
1055 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1056 * @a fEnabledByHost is true).
1057 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1058 * enable VT-x on the host.
1059 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1060 */
1061VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1062 void *pvMsrs)
1063{
1064 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1065 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067
1068 /* Enable VT-x if it's not already enabled by the host. */
1069 if (!fEnabledByHost)
1070 {
1071 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1072 if (RT_FAILURE(rc))
1073 return rc;
1074 }
1075
1076 /*
1077 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1078 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1079 */
1080 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1081 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1082 {
1083 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1084 pCpu->fFlushAsidBeforeUse = false;
1085 }
1086 else
1087 pCpu->fFlushAsidBeforeUse = true;
1088
1089 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1090 ++pCpu->cTlbFlushes;
1091
1092 return VINF_SUCCESS;
1093}
1094
1095
1096/**
1097 * Deactivates VT-x on the current CPU.
1098 *
1099 * @returns VBox status code.
1100 * @param pCpu Pointer to the global CPU info struct.
1101 * @param pvCpuPage Pointer to the VMXON region.
1102 * @param HCPhysCpuPage Physical address of the VMXON region.
1103 *
1104 * @remarks This function should never be called when SUPR0EnableVTx() or
1105 * similar was used to enable VT-x on the host.
1106 */
1107VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1108{
1109 NOREF(pCpu);
1110 NOREF(pvCpuPage);
1111 NOREF(HCPhysCpuPage);
1112
1113 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1114 return hmR0VmxLeaveRootMode();
1115}
1116
1117
1118/**
1119 * Sets the permission bits for the specified MSR in the MSR bitmap.
1120 *
1121 * @param pVCpu Pointer to the VMCPU.
1122 * @param uMSR The MSR value.
1123 * @param enmRead Whether reading this MSR causes a VM-exit.
1124 * @param enmWrite Whether writing this MSR causes a VM-exit.
1125 */
1126static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1127{
1128 int32_t iBit;
1129 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1130
1131 /*
1132 * Layout:
1133 * 0x000 - 0x3ff - Low MSR read bits
1134 * 0x400 - 0x7ff - High MSR read bits
1135 * 0x800 - 0xbff - Low MSR write bits
1136 * 0xc00 - 0xfff - High MSR write bits
1137 */
1138 if (uMsr <= 0x00001FFF)
1139 iBit = uMsr;
1140 else if ( uMsr >= 0xC0000000
1141 && uMsr <= 0xC0001FFF)
1142 {
1143 iBit = (uMsr - 0xC0000000);
1144 pbMsrBitmap += 0x400;
1145 }
1146 else
1147 {
1148 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1149 return;
1150 }
1151
1152 Assert(iBit <= 0x1fff);
1153 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154 ASMBitSet(pbMsrBitmap, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap, iBit);
1157
1158 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160 else
1161 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162}
1163
1164
1165#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1166/**
1167 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1168 * area.
1169 *
1170 * @returns VBox status code.
1171 * @param pVCpu Pointer to the VMCPU.
1172 * @param cMsrs The number of MSRs.
1173 */
1174DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1175{
1176 /* Update the VCPU's copy of the guest MSR count. */
1177 pVCpu->hm.s.vmx.cGuestMsrs = cMsrs;
1178
1179 /* Update number of guest MSRs to load/store across the world-switch. */
1180 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1181 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1182
1183 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1184 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1185 return VINF_SUCCESS;
1186}
1187
1188
1189/**
1190 * Adds a guest/host MSR pair to be swapped during the world-switch as
1191 * part of the auto-load/store MSR area in the VMCS.
1192 *
1193 * @returns VBox status code.
1194 * @param pVCpu Pointer to the VMCPU.
1195 * @param uMsr The MSR.
1196 * @param uGuestMsr Value of the guest MSR.
1197 * @param uHostMsr Value of the host MSR.
1198 */
1199static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, uint64_t uHostMsrValue)
1200{
1201 AssertMsg(HMVMX_MAX_SWAP_MSR_COUNT < MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc),
1202 ("MSR swap count exceeded. Cpu reports %#RX32, our limit %#RX32\n",
1203 MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc), HMVMX_MAX_SWAP_MSR_COUNT));
1204
1205 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1206 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1207 uint32_t i;
1208 for (i = 0; i < cGuestMsrs; i++)
1209 {
1210 if (pGuestMsr->u32Msr == uMsr)
1211 break;
1212 pGuestMsr++;
1213 }
1214
1215 AssertReturn(i < HMVMX_MAX_SWAP_MSR_COUNT, VERR_HM_MSR_SWAP_COUNT_EXCEEDED);
1216 if (i == cGuestMsrs)
1217 {
1218 ++cGuestMsrs;
1219 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1220 if (RT_UNLIKELY(cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc)))
1221 {
1222 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
1223 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1224 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1225 }
1226
1227 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1228 AssertRCReturn(rc, rc);
1229 }
1230
1231 /* Update the MSR values in the auto-load/store MSR area. */
1232 pGuestMsr->u32Msr = uMsr;
1233 pGuestMsr->u64Value = uGuestMsrValue;
1234
1235 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1236 pHostMsr += i;
1237 pHostMsr->u32Msr = uMsr;
1238 pHostMsr->u64Value = uHostMsrValue;
1239
1240 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1241 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
1242 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1243 return VINF_SUCCESS;
1244}
1245
1246
1247/**
1248 * Removes a guest/shost MSR pair to be swapped during the world-switch from the
1249 * auto-load/store MSR area in the VMCS.
1250 *
1251 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1252 * area.
1253 *
1254 * @returns VBox status code.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 */
1258static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cGuestMsrs; i++)
1264 {
1265 /* Find the MSR. */
1266 if (pGuestMsr->u32Msr == uMsr)
1267 {
1268 /* If it's the last MSR, simply reduce the count. */
1269 if (i == cGuestMsrs - 1)
1270 {
1271 --cGuestMsrs;
1272 break;
1273 }
1274
1275 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1276 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1277 pLastGuestMsr += cGuestMsrs;
1278 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1279 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1280
1281 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1282 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1283 pLastHostMsr += cGuestMsrs;
1284 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1285 pHostMsr->u64Value = pLastHostMsr->u64Value;
1286 --cGuestMsrs;
1287 break;
1288 }
1289 pGuestMsr++;
1290 }
1291
1292 /* Update the VMCS if the count changed (meaning the MSR was found). */
1293 if (cGuestMsrs != pVCpu->hm.s.vmx.cGuestMsrs)
1294 {
1295 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1296 AssertRCReturn(rc, rc);
1297 }
1298
1299 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1300 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1301 return VINF_SUCCESS;
1302}
1303
1304
1305/**
1306 * Updates the value of a host MSR in the auto-load/store area in the VMCS.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxUpdateAutoLoadStoreHostMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1316
1317 for (uint32_t i = 0; i < cMsrs; i++)
1318 {
1319 if (pHostMsr->u32Msr == uMsr)
1320 {
1321 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1322 return VINF_SUCCESS;
1323 }
1324 }
1325
1326 return VERR_NOT_FOUND;
1327}
1328
1329#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1330
1331
1332/**
1333 * Flushes the TLB using EPT.
1334 *
1335 * @returns VBox status code.
1336 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1337 * enmFlush).
1338 * @param enmFlush Type of flush.
1339 *
1340 * @remarks Caller is responsible for making sure this function is called only
1341 * when NestedPaging is supported and providing @a enmFlush that is
1342 * supported by the CPU.
1343 * @remarks Can be called with interrupts disabled.
1344 */
1345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1346{
1347 uint64_t au64Descriptor[2];
1348 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1349 au64Descriptor[0] = 0;
1350 else
1351 {
1352 Assert(pVCpu);
1353 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1354 }
1355 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1356
1357 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1358 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1359 rc));
1360 if ( RT_SUCCESS(rc)
1361 && pVCpu)
1362 {
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1364 }
1365}
1366
1367
1368/**
1369 * Flushes the TLB using VPID.
1370 *
1371 * @returns VBox status code.
1372 * @param pVM Pointer to the VM.
1373 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1374 * enmFlush).
1375 * @param enmFlush Type of flush.
1376 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1377 * on @a enmFlush).
1378 *
1379 * @remarks Can be called with interrupts disabled.
1380 */
1381static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1382{
1383 AssertPtr(pVM);
1384 Assert(pVM->hm.s.vmx.fVpid);
1385
1386 uint64_t au64Descriptor[2];
1387 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1388 {
1389 au64Descriptor[0] = 0;
1390 au64Descriptor[1] = 0;
1391 }
1392 else
1393 {
1394 AssertPtr(pVCpu);
1395 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1396 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1397 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1398 au64Descriptor[1] = GCPtr;
1399 }
1400
1401 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1402 AssertMsg(rc == VINF_SUCCESS,
1403 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1404 if ( RT_SUCCESS(rc)
1405 && pVCpu)
1406 {
1407 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1408 }
1409}
1410
1411
1412/**
1413 * Invalidates a guest page by guest virtual address. Only relevant for
1414 * EPT/VPID, otherwise there is nothing really to invalidate.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param GCVirt Guest virtual address of the page to invalidate.
1420 */
1421VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1422{
1423 AssertPtr(pVM);
1424 AssertPtr(pVCpu);
1425 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1426
1427 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1428 if (!fFlushPending)
1429 {
1430 /*
1431 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1432 * See @bugref{6043} and @bugref{6177}.
1433 *
1434 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1435 * function maybe called in a loop with individual addresses.
1436 */
1437 if (pVM->hm.s.vmx.fVpid)
1438 {
1439 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1440 {
1441 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1442 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1443 }
1444 else
1445 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1446 }
1447 else if (pVM->hm.s.fNestedPaging)
1448 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1449 }
1450
1451 return VINF_SUCCESS;
1452}
1453
1454
1455/**
1456 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1457 * otherwise there is nothing really to invalidate.
1458 *
1459 * @returns VBox status code.
1460 * @param pVM Pointer to the VM.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param GCPhys Guest physical address of the page to invalidate.
1463 */
1464VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1465{
1466 LogFlowFunc(("%RGp\n", GCPhys));
1467
1468 /*
1469 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1470 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1471 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1472 */
1473 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1474 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1475 return VINF_SUCCESS;
1476}
1477
1478
1479/**
1480 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1481 * case where neither EPT nor VPID is supported by the CPU.
1482 *
1483 * @param pVM Pointer to the VM.
1484 * @param pVCpu Pointer to the VMCPU.
1485 * @param pCpu Pointer to the global HM struct.
1486 *
1487 * @remarks Called with interrupts disabled.
1488 */
1489static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1490{
1491 AssertPtr(pVCpu);
1492 AssertPtr(pCpu);
1493 NOREF(pVM);
1494
1495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1496
1497 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1498#if 0
1499 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1500 pVCpu->hm.s.TlbShootdown.cPages = 0;
1501#endif
1502
1503 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1504 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1505 pVCpu->hm.s.fForceTLBFlush = false;
1506 return;
1507}
1508
1509
1510/**
1511 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1512 *
1513 * @param pVM Pointer to the VM.
1514 * @param pVCpu Pointer to the VMCPU.
1515 * @param pCpu Pointer to the global HM CPU struct.
1516 * @remarks All references to "ASID" in this function pertains to "VPID" in
1517 * Intel's nomenclature. The reason is, to avoid confusion in compare
1518 * statements since the host-CPU copies are named "ASID".
1519 *
1520 * @remarks Called with interrupts disabled.
1521 */
1522static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1523{
1524#ifdef VBOX_WITH_STATISTICS
1525 bool fTlbFlushed = false;
1526# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1527# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1528 if (!fTlbFlushed) \
1529 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1530 } while (0)
1531#else
1532# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1533# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1534#endif
1535
1536 AssertPtr(pVM);
1537 AssertPtr(pCpu);
1538 AssertPtr(pVCpu);
1539 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1540 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1541 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1542
1543
1544 /*
1545 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1546 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1547 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1548 */
1549 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1550 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1551 {
1552 ++pCpu->uCurrentAsid;
1553 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1554 {
1555 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1556 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1557 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1558 }
1559
1560 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1561 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1562 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1563
1564 /*
1565 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1566 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1567 */
1568 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1569 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1570 HMVMX_SET_TAGGED_TLB_FLUSHED();
1571 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1572 }
1573
1574 /* Check for explicit TLB shootdowns. */
1575 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1576 {
1577 /*
1578 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1579 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1580 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1581 * but not guest-physical mappings.
1582 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1583 */
1584 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1585 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1586 HMVMX_SET_TAGGED_TLB_FLUSHED();
1587 }
1588
1589 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1590 * where it is commented out. Support individual entry flushing
1591 * someday. */
1592#if 0
1593 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1594 {
1595 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1596
1597 /*
1598 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1599 * as supported by the CPU.
1600 */
1601 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1602 {
1603 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1604 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1605 }
1606 else
1607 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1608
1609 HMVMX_SET_TAGGED_TLB_FLUSHED();
1610 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1611 pVCpu->hm.s.TlbShootdown.cPages = 0;
1612 }
1613#endif
1614
1615 pVCpu->hm.s.fForceTLBFlush = false;
1616
1617 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1618
1619 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1620 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1621 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1622 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1623 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1624 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1625 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1626 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1627
1628 /* Update VMCS with the VPID. */
1629 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1630 AssertRC(rc);
1631
1632#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1633}
1634
1635
1636/**
1637 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1638 *
1639 * @returns VBox status code.
1640 * @param pVM Pointer to the VM.
1641 * @param pVCpu Pointer to the VMCPU.
1642 * @param pCpu Pointer to the global HM CPU struct.
1643 *
1644 * @remarks Called with interrupts disabled.
1645 */
1646static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1647{
1648 AssertPtr(pVM);
1649 AssertPtr(pVCpu);
1650 AssertPtr(pCpu);
1651 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1652 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1653
1654 /*
1655 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1656 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1657 */
1658 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1659 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1660 {
1661 pVCpu->hm.s.fForceTLBFlush = true;
1662 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1663 }
1664
1665 /* Check for explicit TLB shootdown flushes. */
1666 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1667 {
1668 pVCpu->hm.s.fForceTLBFlush = true;
1669 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1670 }
1671
1672 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1673 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1674
1675 if (pVCpu->hm.s.fForceTLBFlush)
1676 {
1677 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1678 pVCpu->hm.s.fForceTLBFlush = false;
1679 }
1680 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1681 * where it is commented out. Support individual entry flushing
1682 * someday. */
1683#if 0
1684 else
1685 {
1686 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1687 {
1688 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1689 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1690 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1691 }
1692 else
1693 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1694
1695 pVCpu->hm.s.TlbShootdown.cPages = 0;
1696 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1697 }
1698#endif
1699}
1700
1701
1702/**
1703 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1704 *
1705 * @returns VBox status code.
1706 * @param pVM Pointer to the VM.
1707 * @param pVCpu Pointer to the VMCPU.
1708 * @param pCpu Pointer to the global HM CPU struct.
1709 *
1710 * @remarks Called with interrupts disabled.
1711 */
1712static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1713{
1714 AssertPtr(pVM);
1715 AssertPtr(pVCpu);
1716 AssertPtr(pCpu);
1717 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1718 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1719
1720 /*
1721 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1722 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1723 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1724 */
1725 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1726 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1727 {
1728 pVCpu->hm.s.fForceTLBFlush = true;
1729 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1730 }
1731
1732 /* Check for explicit TLB shootdown flushes. */
1733 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1734 {
1735 /*
1736 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1737 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1738 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1739 */
1740 pVCpu->hm.s.fForceTLBFlush = true;
1741 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1742 }
1743
1744 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1745 if (pVCpu->hm.s.fForceTLBFlush)
1746 {
1747 ++pCpu->uCurrentAsid;
1748 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1749 {
1750 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1751 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1752 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1753 }
1754
1755 pVCpu->hm.s.fForceTLBFlush = false;
1756 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1757 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1758 if (pCpu->fFlushAsidBeforeUse)
1759 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1760 }
1761 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1762 * where it is commented out. Support individual entry flushing
1763 * someday. */
1764#if 0
1765 else
1766 {
1767 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1768 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1769 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1770 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1771
1772 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1773 {
1774 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1776 {
1777 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1778 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1779 }
1780 else
1781 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1782
1783 pVCpu->hm.s.TlbShootdown.cPages = 0;
1784 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1785 }
1786 else
1787 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1788 }
1789#endif
1790
1791 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1792 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1793 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1794 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1795 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1796 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1797
1798 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1799 AssertRC(rc);
1800}
1801
1802
1803/**
1804 * Flushes the guest TLB entry based on CPU capabilities.
1805 *
1806 * @param pVCpu Pointer to the VMCPU.
1807 * @param pCpu Pointer to the global HM CPU struct.
1808 */
1809DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1810{
1811#ifdef HMVMX_ALWAYS_FLUSH_TLB
1812 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1813#endif
1814 PVM pVM = pVCpu->CTX_SUFF(pVM);
1815 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1816 {
1817 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1818 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1819 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1820 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1821 default:
1822 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1823 break;
1824 }
1825
1826 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
1827 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
1828
1829 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
1830}
1831
1832
1833/**
1834 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1835 * TLB entries from the host TLB before VM-entry.
1836 *
1837 * @returns VBox status code.
1838 * @param pVM Pointer to the VM.
1839 */
1840static int hmR0VmxSetupTaggedTlb(PVM pVM)
1841{
1842 /*
1843 * Determine optimal flush type for Nested Paging.
1844 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1845 * guest execution (see hmR3InitFinalizeR0()).
1846 */
1847 if (pVM->hm.s.fNestedPaging)
1848 {
1849 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1850 {
1851 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1852 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1853 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1854 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1855 else
1856 {
1857 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1858 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1859 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1860 }
1861
1862 /* Make sure the write-back cacheable memory type for EPT is supported. */
1863 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1864 {
1865 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1866 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1867 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1868 }
1869 }
1870 else
1871 {
1872 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1873 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1874 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1875 }
1876 }
1877
1878 /*
1879 * Determine optimal flush type for VPID.
1880 */
1881 if (pVM->hm.s.vmx.fVpid)
1882 {
1883 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1884 {
1885 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1886 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1887 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1888 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1889 else
1890 {
1891 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1892 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1893 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1894 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1895 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1896 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1897 pVM->hm.s.vmx.fVpid = false;
1898 }
1899 }
1900 else
1901 {
1902 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1903 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1904 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1905 pVM->hm.s.vmx.fVpid = false;
1906 }
1907 }
1908
1909 /*
1910 * Setup the handler for flushing tagged-TLBs.
1911 */
1912 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1913 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1914 else if (pVM->hm.s.fNestedPaging)
1915 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1916 else if (pVM->hm.s.vmx.fVpid)
1917 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1918 else
1919 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1920 return VINF_SUCCESS;
1921}
1922
1923
1924/**
1925 * Sets up pin-based VM-execution controls in the VMCS.
1926 *
1927 * @returns VBox status code.
1928 * @param pVM Pointer to the VM.
1929 * @param pVCpu Pointer to the VMCPU.
1930 */
1931static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1932{
1933 AssertPtr(pVM);
1934 AssertPtr(pVCpu);
1935
1936 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1937 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1938
1939 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1940 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1941 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1942
1943 /* Enable the VMX preemption timer. */
1944 if (pVM->hm.s.vmx.fUsePreemptTimer)
1945 {
1946 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1947 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1948 }
1949
1950 if ((val & zap) != val)
1951 {
1952 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1953 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1954 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1955 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1956 }
1957
1958 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1959 AssertRCReturn(rc, rc);
1960
1961 /* Update VCPU with the currently set pin-based VM-execution controls. */
1962 pVCpu->hm.s.vmx.u32PinCtls = val;
1963 return rc;
1964}
1965
1966
1967/**
1968 * Sets up processor-based VM-execution controls in the VMCS.
1969 *
1970 * @returns VBox status code.
1971 * @param pVM Pointer to the VM.
1972 * @param pVMCPU Pointer to the VMCPU.
1973 */
1974static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1975{
1976 AssertPtr(pVM);
1977 AssertPtr(pVCpu);
1978
1979 int rc = VERR_INTERNAL_ERROR_5;
1980 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1981 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1982
1983 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1984 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1985 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1986 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1987 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1988 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1989 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1990
1991 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1992 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1993 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1994 {
1995 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1996 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1997 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1998 }
1999
2000 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2001 if (!pVM->hm.s.fNestedPaging)
2002 {
2003 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2004 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2005 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2006 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2007 }
2008
2009 /* Use TPR shadowing if supported by the CPU. */
2010 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2011 {
2012 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2013 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2014 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2015 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2016 AssertRCReturn(rc, rc);
2017
2018 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2019 /* CR8 writes causes a VM-exit based on TPR threshold. */
2020 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2021 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2022 }
2023 else
2024 {
2025 /*
2026 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2027 * Set this control only for 64-bit guests.
2028 */
2029 if (pVM->hm.s.fAllow64BitGuests)
2030 {
2031 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2032 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2033 }
2034 }
2035
2036 /* Use MSR-bitmaps if supported by the CPU. */
2037 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2038 {
2039 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2040
2041 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2042 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2043 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2044 AssertRCReturn(rc, rc);
2045
2046 /*
2047 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2048 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
2049 */
2050 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2051 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2052 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2053 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2054 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2055 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2056 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2057 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2058 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2059 }
2060
2061 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2062 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2063 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2064
2065 if ((val & zap) != val)
2066 {
2067 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2068 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2069 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2070 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2071 }
2072
2073 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2074 AssertRCReturn(rc, rc);
2075
2076 /* Update VCPU with the currently set processor-based VM-execution controls. */
2077 pVCpu->hm.s.vmx.u32ProcCtls = val;
2078
2079 /*
2080 * Secondary processor-based VM-execution controls.
2081 */
2082 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2083 {
2084 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2085 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2086
2087 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2088 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2089
2090 if (pVM->hm.s.fNestedPaging)
2091 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2092 else
2093 {
2094 /*
2095 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2096 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2097 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2098 */
2099 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2100 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2101 }
2102
2103 if (pVM->hm.s.vmx.fVpid)
2104 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2105
2106 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2107 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2108
2109 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2110 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2111 * done dynamically. */
2112 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2113 {
2114 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2115 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2116 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2117 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2118 AssertRCReturn(rc, rc);
2119 }
2120
2121 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2122 {
2123 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2124 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2125 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2126 }
2127
2128 if ((val & zap) != val)
2129 {
2130 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2131 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2132 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2133 }
2134
2135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2136 AssertRCReturn(rc, rc);
2137
2138 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2139 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2140 }
2141 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2142 {
2143 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2144 "available\n"));
2145 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2146 }
2147
2148 return VINF_SUCCESS;
2149}
2150
2151
2152/**
2153 * Sets up miscellaneous (everything other than Pin & Processor-based
2154 * VM-execution) control fields in the VMCS.
2155 *
2156 * @returns VBox status code.
2157 * @param pVM Pointer to the VM.
2158 * @param pVCpu Pointer to the VMCPU.
2159 */
2160static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2161{
2162 AssertPtr(pVM);
2163 AssertPtr(pVCpu);
2164
2165 int rc = VERR_GENERAL_FAILURE;
2166
2167 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2168#if 0
2169 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2170 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2171 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2172
2173 /*
2174 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2175 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2176 * We thus use the exception bitmap to control it rather than use both.
2177 */
2178 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2179 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2180
2181 /** @todo Explore possibility of using IO-bitmaps. */
2182 /* All IO & IOIO instructions cause VM-exits. */
2183 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2184 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2185
2186 /* Initialize the MSR-bitmap area. */
2187 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2188 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2189 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2190#endif
2191
2192#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2193 /* Setup MSR autoloading/storing. */
2194 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2195 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2196 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2197 AssertRCReturn(rc, rc);
2198 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2199 AssertRCReturn(rc, rc);
2200
2201 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2202 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2203 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2204 AssertRCReturn(rc, rc);
2205#endif
2206
2207 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2208 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2209 AssertRCReturn(rc, rc);
2210
2211 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2212#if 0
2213 /* Setup debug controls */
2214 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2215 AssertRCReturn(rc, rc);
2216 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2217 AssertRCReturn(rc, rc);
2218#endif
2219
2220 return rc;
2221}
2222
2223
2224/**
2225 * Sets up the initial exception bitmap in the VMCS based on static conditions
2226 * (i.e. conditions that cannot ever change after starting the VM).
2227 *
2228 * @returns VBox status code.
2229 * @param pVM Pointer to the VM.
2230 * @param pVCpu Pointer to the VMCPU.
2231 */
2232static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2233{
2234 AssertPtr(pVM);
2235 AssertPtr(pVCpu);
2236
2237 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2238
2239 uint32_t u32XcptBitmap = 0;
2240
2241 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2242 if (!pVM->hm.s.fNestedPaging)
2243 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2244
2245 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2246 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2247 AssertRCReturn(rc, rc);
2248 return rc;
2249}
2250
2251
2252/**
2253 * Sets up the initial guest-state mask. The guest-state mask is consulted
2254 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2255 * for the nested virtualization case (as it would cause a VM-exit).
2256 *
2257 * @param pVCpu Pointer to the VMCPU.
2258 */
2259static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2260{
2261 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2262 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Does per-VM VT-x initialization.
2269 *
2270 * @returns VBox status code.
2271 * @param pVM Pointer to the VM.
2272 */
2273VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2274{
2275 LogFlowFunc(("pVM=%p\n", pVM));
2276
2277 int rc = hmR0VmxStructsAlloc(pVM);
2278 if (RT_FAILURE(rc))
2279 {
2280 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2281 return rc;
2282 }
2283
2284 return VINF_SUCCESS;
2285}
2286
2287
2288/**
2289 * Does per-VM VT-x termination.
2290 *
2291 * @returns VBox status code.
2292 * @param pVM Pointer to the VM.
2293 */
2294VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2295{
2296 LogFlowFunc(("pVM=%p\n", pVM));
2297
2298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2299 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2300 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2301#endif
2302 hmR0VmxStructsFree(pVM);
2303 return VINF_SUCCESS;
2304}
2305
2306
2307/**
2308 * Sets up the VM for execution under VT-x.
2309 * This function is only called once per-VM during initialization.
2310 *
2311 * @returns VBox status code.
2312 * @param pVM Pointer to the VM.
2313 */
2314VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2315{
2316 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2317 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2318
2319 LogFlowFunc(("pVM=%p\n", pVM));
2320
2321 /*
2322 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2323 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2324 */
2325 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2326 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2327 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2328 || !pVM->hm.s.vmx.pRealModeTSS))
2329 {
2330 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2331 return VERR_INTERNAL_ERROR;
2332 }
2333
2334#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2335 /*
2336 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2337 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2338 */
2339 if ( pVM->hm.s.fAllow64BitGuests
2340 && !HMVMX_IS_64BIT_HOST_MODE())
2341 {
2342 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2343 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2344 }
2345#endif
2346
2347 /* Initialize these always, see hmR3InitFinalizeR0().*/
2348 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2349 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2350
2351 /* Setup the tagged-TLB flush handlers. */
2352 int rc = hmR0VmxSetupTaggedTlb(pVM);
2353 if (RT_FAILURE(rc))
2354 {
2355 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2356 return rc;
2357 }
2358
2359 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2360 {
2361 PVMCPU pVCpu = &pVM->aCpus[i];
2362 AssertPtr(pVCpu);
2363 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2364
2365 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2366 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2367
2368 /* Set revision dword at the beginning of the VMCS structure. */
2369 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2370
2371 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2372 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2373 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2374 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2375
2376 /* Load this VMCS as the current VMCS. */
2377 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2378 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2379 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2380
2381 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2382 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2383 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2384
2385 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2386 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2387 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2388
2389 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2390 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2391 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2392
2393 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2394 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2395 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2396
2397 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2398 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2399 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2400
2401#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2402 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2403 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2404 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2405#endif
2406
2407 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2408 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2409 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2410 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2411
2412 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2413
2414 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2415 }
2416
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2423 * the VMCS.
2424 *
2425 * @returns VBox status code.
2426 * @param pVM Pointer to the VM.
2427 * @param pVCpu Pointer to the VMCPU.
2428 */
2429DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2430{
2431 RTCCUINTREG uReg = ASMGetCR0();
2432 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2433 AssertRCReturn(rc, rc);
2434
2435#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2436 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2437 if (HMVMX_IS_64BIT_HOST_MODE())
2438 {
2439 uint64_t uRegCR3 = HMR0Get64bitCR3();
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2441 }
2442 else
2443#endif
2444 {
2445 uReg = ASMGetCR3();
2446 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2447 }
2448 AssertRCReturn(rc, rc);
2449
2450 uReg = ASMGetCR4();
2451 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2452 AssertRCReturn(rc, rc);
2453 return rc;
2454}
2455
2456
2457#if HC_ARCH_BITS == 64
2458/**
2459 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2460 * requirements. See hmR0VmxSaveHostSegmentRegs().
2461 */
2462# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2463 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2464 { \
2465 bool fValidSelector = true; \
2466 if ((selValue) & X86_SEL_LDT) \
2467 { \
2468 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2469 fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2470 } \
2471 if (fValidSelector) \
2472 { \
2473 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2474 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2475 } \
2476 (selValue) = 0; \
2477 }
2478#endif
2479
2480
2481/**
2482 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2483 * the host-state area in the VMCS.
2484 *
2485 * @returns VBox status code.
2486 * @param pVM Pointer to the VM.
2487 * @param pVCpu Pointer to the VMCPU.
2488 */
2489DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2490{
2491 int rc = VERR_INTERNAL_ERROR_5;
2492
2493 /*
2494 * Host DS, ES, FS and GS segment registers.
2495 */
2496#if HC_ARCH_BITS == 64
2497 RTSEL uSelDS = ASMGetDS();
2498 RTSEL uSelES = ASMGetES();
2499 RTSEL uSelFS = ASMGetFS();
2500 RTSEL uSelGS = ASMGetGS();
2501#else
2502 RTSEL uSelDS = 0;
2503 RTSEL uSelES = 0;
2504 RTSEL uSelFS = 0;
2505 RTSEL uSelGS = 0;
2506#endif
2507
2508 /* Recalculate which host-state bits need to be manually restored. */
2509 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2510
2511 /*
2512 * Host CS and SS segment registers.
2513 */
2514#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2515 RTSEL uSelCS;
2516 RTSEL uSelSS;
2517 if (HMVMX_IS_64BIT_HOST_MODE())
2518 {
2519 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2520 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2521 }
2522 else
2523 {
2524 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2525 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2526 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2527 }
2528#else
2529 RTSEL uSelCS = ASMGetCS();
2530 RTSEL uSelSS = ASMGetSS();
2531#endif
2532
2533 /*
2534 * Host TR segment register.
2535 */
2536 RTSEL uSelTR = ASMGetTR();
2537
2538#if HC_ARCH_BITS == 64
2539 /*
2540 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2541 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2542 */
2543 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2544 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2545 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2546 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2547# undef VMXLOCAL_ADJUST_HOST_SEG
2548#endif
2549
2550 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2551 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2552 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2553 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2554 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2555 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2556 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2557 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2558 Assert(uSelCS);
2559 Assert(uSelTR);
2560
2561 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2562#if 0
2563 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2564 Assert(uSelSS != 0);
2565#endif
2566
2567 /* Write these host selector fields into the host-state area in the VMCS. */
2568 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2570#if HC_ARCH_BITS == 64
2571 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2575#endif
2576 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2577
2578 /*
2579 * Host GDTR and IDTR.
2580 */
2581 RTGDTR Gdtr;
2582 RT_ZERO(Gdtr);
2583#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2584 if (HMVMX_IS_64BIT_HOST_MODE())
2585 {
2586 X86XDTR64 Gdtr64;
2587 X86XDTR64 Idtr64;
2588 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2589 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2591
2592 Gdtr.cbGdt = Gdtr64.cb;
2593 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2594 }
2595 else
2596#endif
2597 {
2598 RTIDTR Idtr;
2599 ASMGetGDTR(&Gdtr);
2600 ASMGetIDTR(&Idtr);
2601 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2602 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2603
2604#if HC_ARCH_BITS == 64
2605 /*
2606 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2607 * maximum limit (0xffff) on every VM-exit.
2608 */
2609 if (Gdtr.cbGdt != 0xffff)
2610 {
2611 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2612 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2613 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2614 }
2615
2616 /*
2617 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2618 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2619 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2620 */
2621 if (Idtr.cbIdt < 0x0fff)
2622 {
2623 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2624 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2625 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2626 }
2627#endif
2628 }
2629
2630 /*
2631 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2632 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2633 */
2634 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2635 {
2636 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2637 return VERR_VMX_INVALID_HOST_STATE;
2638 }
2639
2640 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2641#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2642 if (HMVMX_IS_64BIT_HOST_MODE())
2643 {
2644 /* We need the 64-bit TR base for hybrid darwin. */
2645 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2646 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2647 }
2648 else
2649#endif
2650 {
2651 uintptr_t uTRBase;
2652#if HC_ARCH_BITS == 64
2653 uTRBase = X86DESC64_BASE(pDesc);
2654
2655 /*
2656 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2657 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2658 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2659 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2660 *
2661 * [1] See Intel spec. 3.5 "System Descriptor Types".
2662 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2663 */
2664 Assert(pDesc->System.u4Type == 11);
2665 if ( pDesc->System.u16LimitLow != 0x67
2666 || pDesc->System.u4LimitHigh)
2667 {
2668 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2669 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2670
2671 /* Store the GDTR here as we need it while restoring TR. */
2672 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2673 }
2674#else
2675 uTRBase = X86DESC_BASE(pDesc);
2676#endif
2677 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2678 }
2679 AssertRCReturn(rc, rc);
2680
2681 /*
2682 * Host FS base and GS base.
2683 */
2684#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2685 if (HMVMX_IS_64BIT_HOST_MODE())
2686 {
2687 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2688 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2689 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2690 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2691
2692# if HC_ARCH_BITS == 64
2693 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2694 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2695 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2696 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2697 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2698# endif
2699 }
2700#endif
2701 return rc;
2702}
2703
2704
2705/**
2706 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2707 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2708 * the host after every successful VM exit.
2709 *
2710 * @returns VBox status code.
2711 * @param pVM Pointer to the VM.
2712 * @param pVCpu Pointer to the VMCPU.
2713 */
2714DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2715{
2716 AssertPtr(pVCpu);
2717 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2718
2719 int rc = VINF_SUCCESS;
2720#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2721 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2722 uint32_t cHostMsrs = 0;
2723 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2724
2725 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2726 {
2727 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2728
2729# if HC_ARCH_BITS == 64
2730 /* Paranoia. 64-bit code requires these bits to be set always. */
2731 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2732
2733 /*
2734 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2735 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2736 * some reason (e.g. allow transparent reads) we would activate the code below.
2737 */
2738# if 0
2739 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2740 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2741 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2742 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2743 if (CPUMIsGuestInLongMode(pVCpu))
2744 {
2745 uint64_t u64GuestEfer;
2746 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2747 AssertRC(rc);
2748
2749 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2750 {
2751 pHostMsr->u32Msr = MSR_K6_EFER;
2752 pHostMsr->u32Reserved = 0;
2753 pHostMsr->u64Value = u64HostEfer;
2754 pHostMsr++; cHostMsrs++;
2755 }
2756 }
2757# endif
2758# else /* HC_ARCH_BITS != 64 */
2759 pHostMsr->u32Msr = MSR_K6_EFER;
2760 pHostMsr->u32Reserved = 0;
2761# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2762 if (CPUMIsGuestInLongMode(pVCpu))
2763 {
2764 /* Must match the EFER value in our 64 bits switcher. */
2765 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2766 }
2767 else
2768# endif
2769 pHostMsr->u64Value = u64HostEfer;
2770 pHostMsr++; cHostMsrs++;
2771# endif /* HC_ARCH_BITS == 64 */
2772 }
2773
2774# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2775 if (HMVMX_IS_64BIT_HOST_MODE())
2776 {
2777 pHostMsr->u32Msr = MSR_K6_STAR;
2778 pHostMsr->u32Reserved = 0;
2779 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2780 pHostMsr++; cHostMsrs++;
2781 pHostMsr->u32Msr = MSR_K8_LSTAR;
2782 pHostMsr->u32Reserved = 0;
2783 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2784 pHostMsr++; cHostMsrs++;
2785 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2786 pHostMsr->u32Reserved = 0;
2787 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2788 pHostMsr++; cHostMsrs++;
2789 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2790 pHostMsr->u32Reserved = 0;
2791 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2792 pHostMsr++; cHostMsrs++;
2793 }
2794# endif
2795
2796 /* Host TSC AUX MSR must be restored since we always load/store guest TSC AUX MSR. */
2797 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2798 {
2799 pHostMsr->u32Msr = MSR_K8_TSC_AUX;
2800 pHostMsr->u32Reserved = 0;
2801 pHostMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
2802 pHostMsr++; cHostMsrs++;
2803 }
2804
2805 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2806 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2807 {
2808 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2809 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2810 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2811 }
2812
2813 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2814#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2815
2816 /*
2817 * Host Sysenter MSRs.
2818 */
2819 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2820 AssertRCReturn(rc, rc);
2821#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2822 if (HMVMX_IS_64BIT_HOST_MODE())
2823 {
2824 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2825 AssertRCReturn(rc, rc);
2826 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2827 }
2828 else
2829 {
2830 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2831 AssertRCReturn(rc, rc);
2832 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2833 }
2834#elif HC_ARCH_BITS == 32
2835 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2836 AssertRCReturn(rc, rc);
2837 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2838#else
2839 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2840 AssertRCReturn(rc, rc);
2841 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2842#endif
2843 AssertRCReturn(rc, rc);
2844
2845 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2846 * hmR0VmxSetupExitCtls() !! */
2847 return rc;
2848}
2849
2850
2851/**
2852 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2853 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2854 * controls".
2855 *
2856 * @returns VBox status code.
2857 * @param pVCpu Pointer to the VMCPU.
2858 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2859 * out-of-sync. Make sure to update the required fields
2860 * before using them.
2861 *
2862 * @remarks No-long-jump zone!!!
2863 */
2864DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2865{
2866 int rc = VINF_SUCCESS;
2867 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2868 {
2869 PVM pVM = pVCpu->CTX_SUFF(pVM);
2870 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2871 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2872
2873 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2874 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2875
2876 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2877 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2878 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2879 else
2880 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2881
2882 /*
2883 * The following should -not- be set (since we're not in SMM mode):
2884 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2885 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2886 */
2887
2888 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2889 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2890 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2891
2892 if ((val & zap) != val)
2893 {
2894 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2895 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2896 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2897 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2898 }
2899
2900 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2901 AssertRCReturn(rc, rc);
2902
2903 /* Update VCPU with the currently set VM-exit controls. */
2904 pVCpu->hm.s.vmx.u32EntryCtls = val;
2905 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2906 }
2907 return rc;
2908}
2909
2910
2911/**
2912 * Sets up the VM-exit controls in the VMCS.
2913 *
2914 * @returns VBox status code.
2915 * @param pVM Pointer to the VM.
2916 * @param pVCpu Pointer to the VMCPU.
2917 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2918 * out-of-sync. Make sure to update the required fields
2919 * before using them.
2920 *
2921 * @remarks requires EFER.
2922 */
2923DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2924{
2925 int rc = VINF_SUCCESS;
2926 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2927 {
2928 PVM pVM = pVCpu->CTX_SUFF(pVM);
2929 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2930 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2931
2932 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2933 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2934
2935 /*
2936 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2937 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2938 */
2939#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2940 if (HMVMX_IS_64BIT_HOST_MODE())
2941 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2942 else
2943 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2944#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2945 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2946 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2947 else
2948 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2949#endif
2950
2951 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2952 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2953
2954 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2955 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2956 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2957 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2958 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2959
2960 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2961 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2962
2963 if ((val & zap) != val)
2964 {
2965 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2966 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2967 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2968 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2969 }
2970
2971 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2972 AssertRCReturn(rc, rc);
2973
2974 /* Update VCPU with the currently set VM-exit controls. */
2975 pVCpu->hm.s.vmx.u32ExitCtls = val;
2976 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2977 }
2978 return rc;
2979}
2980
2981
2982/**
2983 * Loads the guest APIC and related state.
2984 *
2985 * @returns VBox status code.
2986 * @param pVM Pointer to the VM.
2987 * @param pVCpu Pointer to the VMCPU.
2988 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2989 * out-of-sync. Make sure to update the required fields
2990 * before using them.
2991 */
2992DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2993{
2994 int rc = VINF_SUCCESS;
2995 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2996 {
2997 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2998 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2999 {
3000 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3001
3002 bool fPendingIntr = false;
3003 uint8_t u8Tpr = 0;
3004 uint8_t u8PendingIntr = 0;
3005 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3006 AssertRCReturn(rc, rc);
3007
3008 /*
3009 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3010 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3011 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3012 * the interrupt when we VM-exit for other reasons.
3013 */
3014 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3015 uint32_t u32TprThreshold = 0;
3016 if (fPendingIntr)
3017 {
3018 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3019 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3020 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3021 if (u8PendingPriority <= u8TprPriority)
3022 u32TprThreshold = u8PendingPriority;
3023 else
3024 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3025 }
3026 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3027
3028 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3029 AssertRCReturn(rc, rc);
3030 }
3031
3032 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3033 }
3034 return rc;
3035}
3036
3037
3038/**
3039 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3040 *
3041 * @returns Guest's interruptibility-state.
3042 * @param pVCpu Pointer to the VMCPU.
3043 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3044 * out-of-sync. Make sure to update the required fields
3045 * before using them.
3046 *
3047 * @remarks No-long-jump zone!!!
3048 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3049 */
3050DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3051{
3052 /*
3053 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3054 * inhibit interrupts or clear any existing interrupt-inhibition.
3055 */
3056 uint32_t uIntrState = 0;
3057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3058 {
3059 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3060 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
3061 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3062 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3063 {
3064 /*
3065 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3066 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3067 */
3068 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3069 }
3070 else if (pMixedCtx->eflags.Bits.u1IF)
3071 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3072 else
3073 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3074 }
3075 return uIntrState;
3076}
3077
3078
3079/**
3080 * Loads the guest's interruptibility-state into the guest-state area in the
3081 * VMCS.
3082 *
3083 * @returns VBox status code.
3084 * @param pVCpu Pointer to the VMCPU.
3085 * @param uIntrState The interruptibility-state to set.
3086 */
3087static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3088{
3089 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3090 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3091 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3092 AssertRCReturn(rc, rc);
3093 return rc;
3094}
3095
3096
3097/**
3098 * Loads the guest's RIP into the guest-state area in the VMCS.
3099 *
3100 * @returns VBox status code.
3101 * @param pVCpu Pointer to the VMCPU.
3102 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3103 * out-of-sync. Make sure to update the required fields
3104 * before using them.
3105 *
3106 * @remarks No-long-jump zone!!!
3107 */
3108static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3109{
3110 int rc = VINF_SUCCESS;
3111 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3112 {
3113 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3114 AssertRCReturn(rc, rc);
3115
3116 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3117 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3118 }
3119 return rc;
3120}
3121
3122
3123/**
3124 * Loads the guest's RSP into the guest-state area in the VMCS.
3125 *
3126 * @returns VBox status code.
3127 * @param pVCpu Pointer to the VMCPU.
3128 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3129 * out-of-sync. Make sure to update the required fields
3130 * before using them.
3131 *
3132 * @remarks No-long-jump zone!!!
3133 */
3134static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3135{
3136 int rc = VINF_SUCCESS;
3137 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3138 {
3139 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3140 AssertRCReturn(rc, rc);
3141
3142 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3143 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3144 }
3145 return rc;
3146}
3147
3148
3149/**
3150 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3151 *
3152 * @returns VBox status code.
3153 * @param pVCpu Pointer to the VMCPU.
3154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3155 * out-of-sync. Make sure to update the required fields
3156 * before using them.
3157 *
3158 * @remarks No-long-jump zone!!!
3159 */
3160static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3161{
3162 int rc = VINF_SUCCESS;
3163 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3164 {
3165 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3166 Let us assert it as such and use 32-bit VMWRITE. */
3167 Assert(!(pMixedCtx->rflags.u64 >> 32));
3168 X86EFLAGS Eflags = pMixedCtx->eflags;
3169 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3170 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3171
3172 /*
3173 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3174 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3175 */
3176 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3177 {
3178 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3179 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3180 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3181 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3182 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3183 }
3184
3185 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3186 AssertRCReturn(rc, rc);
3187
3188 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3189 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3190 }
3191 return rc;
3192}
3193
3194
3195/**
3196 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3197 *
3198 * @returns VBox status code.
3199 * @param pVCpu Pointer to the VMCPU.
3200 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201 * out-of-sync. Make sure to update the required fields
3202 * before using them.
3203 *
3204 * @remarks No-long-jump zone!!!
3205 */
3206DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3207{
3208 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3209 AssertRCReturn(rc, rc);
3210 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3211 AssertRCReturn(rc, rc);
3212 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3213 AssertRCReturn(rc, rc);
3214 return rc;
3215}
3216
3217
3218/**
3219 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3220 * CR0 is partially shared with the host and we have to consider the FPU bits.
3221 *
3222 * @returns VBox status code.
3223 * @param pVM Pointer to the VM.
3224 * @param pVCpu Pointer to the VMCPU.
3225 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3226 * out-of-sync. Make sure to update the required fields
3227 * before using them.
3228 *
3229 * @remarks No-long-jump zone!!!
3230 */
3231static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3232{
3233 /*
3234 * Guest CR0.
3235 * Guest FPU.
3236 */
3237 int rc = VINF_SUCCESS;
3238 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3239 {
3240 Assert(!(pMixedCtx->cr0 >> 32));
3241 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3242 PVM pVM = pVCpu->CTX_SUFF(pVM);
3243
3244 /* The guest's view (read access) of its CR0 is unblemished. */
3245 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3246 AssertRCReturn(rc, rc);
3247 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3248
3249 /* Setup VT-x's view of the guest CR0. */
3250 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3251 if (pVM->hm.s.fNestedPaging)
3252 {
3253 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3254 {
3255 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3256 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3257 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3258 }
3259 else
3260 {
3261 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3262 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3263 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3264 }
3265
3266 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3267 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3268 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3269
3270 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3271 AssertRCReturn(rc, rc);
3272 }
3273 else
3274 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3275
3276 /*
3277 * Guest FPU bits.
3278 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3279 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3280 */
3281 u32GuestCR0 |= X86_CR0_NE;
3282 bool fInterceptNM = false;
3283 if (CPUMIsGuestFPUStateActive(pVCpu))
3284 {
3285 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3286 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3287 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3288 }
3289 else
3290 {
3291 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3292 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3293 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3294 }
3295
3296 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3297 bool fInterceptMF = false;
3298 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3299 fInterceptMF = true;
3300
3301 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3302 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3303 {
3304 Assert(PDMVmmDevHeapIsEnabled(pVM));
3305 Assert(pVM->hm.s.vmx.pRealModeTSS);
3306 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3307 fInterceptNM = true;
3308 fInterceptMF = true;
3309 }
3310 else
3311 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3312
3313 if (fInterceptNM)
3314 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3315 else
3316 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3317
3318 if (fInterceptMF)
3319 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3320 else
3321 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3322
3323 /* Additional intercepts for debugging, define these yourself explicitly. */
3324#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3325 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3326 | RT_BIT(X86_XCPT_BP)
3327 | RT_BIT(X86_XCPT_DB)
3328 | RT_BIT(X86_XCPT_DE)
3329 | RT_BIT(X86_XCPT_NM)
3330 | RT_BIT(X86_XCPT_UD)
3331 | RT_BIT(X86_XCPT_NP)
3332 | RT_BIT(X86_XCPT_SS)
3333 | RT_BIT(X86_XCPT_GP)
3334 | RT_BIT(X86_XCPT_PF)
3335 | RT_BIT(X86_XCPT_MF)
3336 ;
3337#elif defined(HMVMX_ALWAYS_TRAP_PF)
3338 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3339#endif
3340
3341 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3342
3343 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3344 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3345 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3346 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3347 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3348 else
3349 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3350
3351 u32GuestCR0 |= uSetCR0;
3352 u32GuestCR0 &= uZapCR0;
3353 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3354
3355 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3356 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3357 AssertRCReturn(rc, rc);
3358 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3359 AssertRCReturn(rc, rc);
3360 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3361
3362 /*
3363 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3364 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3365 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3366 */
3367 uint32_t u32CR0Mask = 0;
3368 u32CR0Mask = X86_CR0_PE
3369 | X86_CR0_NE
3370 | X86_CR0_WP
3371 | X86_CR0_PG
3372 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3373 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3374 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3375
3376 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3377 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3378 * and @bugref{6944}. */
3379#if 0
3380 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3381 u32CR0Mask &= ~X86_CR0_PE;
3382#endif
3383 if (pVM->hm.s.fNestedPaging)
3384 u32CR0Mask &= ~X86_CR0_WP;
3385
3386 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3387 if (fInterceptNM)
3388 {
3389 u32CR0Mask |= X86_CR0_TS
3390 | X86_CR0_MP;
3391 }
3392
3393 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3394 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3395 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3396 AssertRCReturn(rc, rc);
3397 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3398
3399 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3400 }
3401 return rc;
3402}
3403
3404
3405/**
3406 * Loads the guest control registers (CR3, CR4) into the guest-state area
3407 * in the VMCS.
3408 *
3409 * @returns VBox status code.
3410 * @param pVM Pointer to the VM.
3411 * @param pVCpu Pointer to the VMCPU.
3412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3413 * out-of-sync. Make sure to update the required fields
3414 * before using them.
3415 *
3416 * @remarks No-long-jump zone!!!
3417 */
3418static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3419{
3420 int rc = VINF_SUCCESS;
3421 PVM pVM = pVCpu->CTX_SUFF(pVM);
3422
3423 /*
3424 * Guest CR2.
3425 * It's always loaded in the assembler code. Nothing to do here.
3426 */
3427
3428 /*
3429 * Guest CR3.
3430 */
3431 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3432 {
3433 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3434 if (pVM->hm.s.fNestedPaging)
3435 {
3436 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3437
3438 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3439 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3440 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3441 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3442
3443 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3444 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3445 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3446
3447 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3448 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3449 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3450 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3451
3452 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3453 AssertRCReturn(rc, rc);
3454 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3455
3456 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3457 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3458 {
3459 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3460 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3461 {
3462 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3463 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3464 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3465 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3466 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3467 }
3468
3469 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3470 have Unrestricted Execution to handle the guest when it's not using paging. */
3471 GCPhysGuestCR3 = pMixedCtx->cr3;
3472 }
3473 else
3474 {
3475 /*
3476 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3477 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3478 * EPT takes care of translating it to host-physical addresses.
3479 */
3480 RTGCPHYS GCPhys;
3481 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3482 Assert(PDMVmmDevHeapIsEnabled(pVM));
3483
3484 /* We obtain it here every time as the guest could have relocated this PCI region. */
3485 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3486 AssertRCReturn(rc, rc);
3487
3488 GCPhysGuestCR3 = GCPhys;
3489 }
3490
3491 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3492 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3493 }
3494 else
3495 {
3496 /* Non-nested paging case, just use the hypervisor's CR3. */
3497 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3498
3499 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3500 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3501 }
3502 AssertRCReturn(rc, rc);
3503
3504 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3505 }
3506
3507 /*
3508 * Guest CR4.
3509 */
3510 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3511 {
3512 Assert(!(pMixedCtx->cr4 >> 32));
3513 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3514
3515 /* The guest's view of its CR4 is unblemished. */
3516 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3517 AssertRCReturn(rc, rc);
3518 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3519
3520 /* Setup VT-x's view of the guest CR4. */
3521 /*
3522 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3523 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3524 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3525 */
3526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3527 {
3528 Assert(pVM->hm.s.vmx.pRealModeTSS);
3529 Assert(PDMVmmDevHeapIsEnabled(pVM));
3530 u32GuestCR4 &= ~X86_CR4_VME;
3531 }
3532
3533 if (pVM->hm.s.fNestedPaging)
3534 {
3535 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3536 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3537 {
3538 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3539 u32GuestCR4 |= X86_CR4_PSE;
3540 /* Our identity mapping is a 32 bits page directory. */
3541 u32GuestCR4 &= ~X86_CR4_PAE;
3542 }
3543 /* else use guest CR4.*/
3544 }
3545 else
3546 {
3547 /*
3548 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3549 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3550 */
3551 switch (pVCpu->hm.s.enmShadowMode)
3552 {
3553 case PGMMODE_REAL: /* Real-mode. */
3554 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3555 case PGMMODE_32_BIT: /* 32-bit paging. */
3556 {
3557 u32GuestCR4 &= ~X86_CR4_PAE;
3558 break;
3559 }
3560
3561 case PGMMODE_PAE: /* PAE paging. */
3562 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3563 {
3564 u32GuestCR4 |= X86_CR4_PAE;
3565 break;
3566 }
3567
3568 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3569 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3570#ifdef VBOX_ENABLE_64_BITS_GUESTS
3571 break;
3572#endif
3573 default:
3574 AssertFailed();
3575 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3576 }
3577 }
3578
3579 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3580 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3581 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3582 u32GuestCR4 |= uSetCR4;
3583 u32GuestCR4 &= uZapCR4;
3584
3585 /* Write VT-x's view of the guest CR4 into the VMCS. */
3586 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3587 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3588 AssertRCReturn(rc, rc);
3589
3590 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3591 uint32_t u32CR4Mask = 0;
3592 u32CR4Mask = X86_CR4_VME
3593 | X86_CR4_PAE
3594 | X86_CR4_PGE
3595 | X86_CR4_PSE
3596 | X86_CR4_VMXE;
3597 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3598 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3599 AssertRCReturn(rc, rc);
3600
3601 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3602 }
3603 return rc;
3604}
3605
3606
3607/**
3608 * Loads the guest debug registers into the guest-state area in the VMCS.
3609 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3610 *
3611 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu Pointer to the VMCPU.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3624 return VINF_SUCCESS;
3625
3626#ifdef VBOX_STRICT
3627 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3628 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3629 {
3630 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3631 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3632 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3633 }
3634#endif
3635
3636 int rc;
3637 PVM pVM = pVCpu->CTX_SUFF(pVM);
3638 bool fInterceptDB = false;
3639 bool fInterceptMovDRx = false;
3640 if ( pVCpu->hm.s.fSingleInstruction
3641 || DBGFIsStepping(pVCpu))
3642 {
3643 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3644 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3645 {
3646 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3647 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3648 AssertRCReturn(rc, rc);
3649 Assert(fInterceptDB == false);
3650 }
3651 else
3652 {
3653 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3654 pVCpu->hm.s.fClearTrapFlag = true;
3655 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3656 fInterceptDB = true;
3657 }
3658 }
3659
3660 if ( fInterceptDB
3661 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3662 {
3663 /*
3664 * Use the combined guest and host DRx values found in the hypervisor
3665 * register set because the debugger has breakpoints active or someone
3666 * is single stepping on the host side without a monitor trap flag.
3667 *
3668 * Note! DBGF expects a clean DR6 state before executing guest code.
3669 */
3670#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3671 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3672 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3673 {
3674 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3675 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3676 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3677 }
3678 else
3679#endif
3680 if (!CPUMIsHyperDebugStateActive(pVCpu))
3681 {
3682 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3683 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3684 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3685 }
3686
3687 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3688 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3689 AssertRCReturn(rc, rc);
3690
3691 pVCpu->hm.s.fUsingHyperDR7 = true;
3692 fInterceptDB = true;
3693 fInterceptMovDRx = true;
3694 }
3695 else
3696 {
3697 /*
3698 * If the guest has enabled debug registers, we need to load them prior to
3699 * executing guest code so they'll trigger at the right time.
3700 */
3701 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3702 {
3703#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3704 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3705 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3706 {
3707 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3708 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3709 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3710 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3711 }
3712 else
3713#endif
3714 if (!CPUMIsGuestDebugStateActive(pVCpu))
3715 {
3716 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3717 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3718 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3719 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3720 }
3721 Assert(!fInterceptDB);
3722 Assert(!fInterceptMovDRx);
3723 }
3724 /*
3725 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3726 * must intercept #DB in order to maintain a correct DR6 guest value.
3727 */
3728#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3729 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3730 && !CPUMIsGuestDebugStateActive(pVCpu))
3731#else
3732 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3733#endif
3734 {
3735 fInterceptMovDRx = true;
3736 fInterceptDB = true;
3737 }
3738
3739 /* Update guest DR7. */
3740 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3741 AssertRCReturn(rc, rc);
3742
3743 pVCpu->hm.s.fUsingHyperDR7 = false;
3744 }
3745
3746 /*
3747 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3748 */
3749 if (fInterceptDB)
3750 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3751 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3752 {
3753#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3754 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3755#endif
3756 }
3757 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3758 AssertRCReturn(rc, rc);
3759
3760 /*
3761 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3762 */
3763 if (fInterceptMovDRx)
3764 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3765 else
3766 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3767 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3768 AssertRCReturn(rc, rc);
3769
3770 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3771 return VINF_SUCCESS;
3772}
3773
3774
3775#ifdef VBOX_STRICT
3776/**
3777 * Strict function to validate segment registers.
3778 *
3779 * @remarks ASSUMES CR0 is up to date.
3780 */
3781static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3782{
3783 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3784 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3785 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3786 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3787 && ( !CPUMIsGuestInRealModeEx(pCtx)
3788 && !CPUMIsGuestInV86ModeEx(pCtx)))
3789 {
3790 /* Protected mode checks */
3791 /* CS */
3792 Assert(pCtx->cs.Attr.n.u1Present);
3793 Assert(!(pCtx->cs.Attr.u & 0xf00));
3794 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3795 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3796 || !(pCtx->cs.Attr.n.u1Granularity));
3797 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3798 || (pCtx->cs.Attr.n.u1Granularity));
3799 /* CS cannot be loaded with NULL in protected mode. */
3800 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3801 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3802 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3803 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3804 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3805 else
3806 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3807 /* SS */
3808 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3809 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3810 if ( !(pCtx->cr0 & X86_CR0_PE)
3811 || pCtx->cs.Attr.n.u4Type == 3)
3812 {
3813 Assert(!pCtx->ss.Attr.n.u2Dpl);
3814 }
3815 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3816 {
3817 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3818 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3819 Assert(pCtx->ss.Attr.n.u1Present);
3820 Assert(!(pCtx->ss.Attr.u & 0xf00));
3821 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3822 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3823 || !(pCtx->ss.Attr.n.u1Granularity));
3824 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3825 || (pCtx->ss.Attr.n.u1Granularity));
3826 }
3827 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3828 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3829 {
3830 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3831 Assert(pCtx->ds.Attr.n.u1Present);
3832 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3833 Assert(!(pCtx->ds.Attr.u & 0xf00));
3834 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3835 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3836 || !(pCtx->ds.Attr.n.u1Granularity));
3837 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3838 || (pCtx->ds.Attr.n.u1Granularity));
3839 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3840 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3841 }
3842 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3843 {
3844 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3845 Assert(pCtx->es.Attr.n.u1Present);
3846 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3847 Assert(!(pCtx->es.Attr.u & 0xf00));
3848 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3849 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3850 || !(pCtx->es.Attr.n.u1Granularity));
3851 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3852 || (pCtx->es.Attr.n.u1Granularity));
3853 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3854 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3855 }
3856 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3857 {
3858 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3859 Assert(pCtx->fs.Attr.n.u1Present);
3860 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3861 Assert(!(pCtx->fs.Attr.u & 0xf00));
3862 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3863 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3864 || !(pCtx->fs.Attr.n.u1Granularity));
3865 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3866 || (pCtx->fs.Attr.n.u1Granularity));
3867 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3868 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3869 }
3870 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3871 {
3872 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3873 Assert(pCtx->gs.Attr.n.u1Present);
3874 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3875 Assert(!(pCtx->gs.Attr.u & 0xf00));
3876 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3877 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3878 || !(pCtx->gs.Attr.n.u1Granularity));
3879 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3880 || (pCtx->gs.Attr.n.u1Granularity));
3881 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3882 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3883 }
3884 /* 64-bit capable CPUs. */
3885# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3886 Assert(!(pCtx->cs.u64Base >> 32));
3887 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3888 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3889 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3890# endif
3891 }
3892 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3893 || ( CPUMIsGuestInRealModeEx(pCtx)
3894 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3895 {
3896 /* Real and v86 mode checks. */
3897 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3898 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3899 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3900 {
3901 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3902 }
3903 else
3904 {
3905 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3906 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3907 }
3908
3909 /* CS */
3910 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3911 Assert(pCtx->cs.u32Limit == 0xffff);
3912 Assert(u32CSAttr == 0xf3);
3913 /* SS */
3914 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3915 Assert(pCtx->ss.u32Limit == 0xffff);
3916 Assert(u32SSAttr == 0xf3);
3917 /* DS */
3918 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3919 Assert(pCtx->ds.u32Limit == 0xffff);
3920 Assert(u32DSAttr == 0xf3);
3921 /* ES */
3922 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3923 Assert(pCtx->es.u32Limit == 0xffff);
3924 Assert(u32ESAttr == 0xf3);
3925 /* FS */
3926 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3927 Assert(pCtx->fs.u32Limit == 0xffff);
3928 Assert(u32FSAttr == 0xf3);
3929 /* GS */
3930 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3931 Assert(pCtx->gs.u32Limit == 0xffff);
3932 Assert(u32GSAttr == 0xf3);
3933 /* 64-bit capable CPUs. */
3934# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3935 Assert(!(pCtx->cs.u64Base >> 32));
3936 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3937 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3938 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3939# endif
3940 }
3941}
3942#endif /* VBOX_STRICT */
3943
3944
3945/**
3946 * Writes a guest segment register into the guest-state area in the VMCS.
3947 *
3948 * @returns VBox status code.
3949 * @param pVCpu Pointer to the VMCPU.
3950 * @param idxSel Index of the selector in the VMCS.
3951 * @param idxLimit Index of the segment limit in the VMCS.
3952 * @param idxBase Index of the segment base in the VMCS.
3953 * @param idxAccess Index of the access rights of the segment in the VMCS.
3954 * @param pSelReg Pointer to the segment selector.
3955 * @param pCtx Pointer to the guest-CPU context.
3956 *
3957 * @remarks No-long-jump zone!!!
3958 */
3959static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3960 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3961{
3962 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3963 AssertRCReturn(rc, rc);
3964 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3965 AssertRCReturn(rc, rc);
3966 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3967 AssertRCReturn(rc, rc);
3968
3969 uint32_t u32Access = pSelReg->Attr.u;
3970 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3971 {
3972 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3973 u32Access = 0xf3;
3974 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3975 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3976 }
3977 else
3978 {
3979 /*
3980 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3981 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3982 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3983 * loaded in protected-mode have their attribute as 0.
3984 */
3985 if (!u32Access)
3986 u32Access = X86DESCATTR_UNUSABLE;
3987 }
3988
3989 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3990 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3991 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3992
3993 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3994 AssertRCReturn(rc, rc);
3995 return rc;
3996}
3997
3998
3999/**
4000 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4001 * into the guest-state area in the VMCS.
4002 *
4003 * @returns VBox status code.
4004 * @param pVM Pointer to the VM.
4005 * @param pVCPU Pointer to the VMCPU.
4006 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4007 * out-of-sync. Make sure to update the required fields
4008 * before using them.
4009 *
4010 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4011 * @remarks No-long-jump zone!!!
4012 */
4013static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4014{
4015 int rc = VERR_INTERNAL_ERROR_5;
4016 PVM pVM = pVCpu->CTX_SUFF(pVM);
4017
4018 /*
4019 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4020 */
4021 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4022 {
4023 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4024 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4025 {
4026 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4027 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4028 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4029 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4030 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4031 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4032 }
4033
4034#ifdef VBOX_WITH_REM
4035 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4036 {
4037 Assert(pVM->hm.s.vmx.pRealModeTSS);
4038 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4039 if ( pVCpu->hm.s.vmx.fWasInRealMode
4040 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4041 {
4042 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4043 in real-mode (e.g. OpenBSD 4.0) */
4044 REMFlushTBs(pVM);
4045 Log4(("Load: Switch to protected mode detected!\n"));
4046 pVCpu->hm.s.vmx.fWasInRealMode = false;
4047 }
4048 }
4049#endif
4050 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4051 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
4052 AssertRCReturn(rc, rc);
4053 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4054 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
4055 AssertRCReturn(rc, rc);
4056 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4057 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
4058 AssertRCReturn(rc, rc);
4059 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4060 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
4061 AssertRCReturn(rc, rc);
4062 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4063 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
4064 AssertRCReturn(rc, rc);
4065 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4066 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
4067 AssertRCReturn(rc, rc);
4068
4069#ifdef VBOX_STRICT
4070 /* Validate. */
4071 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4072#endif
4073
4074 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4075 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4076 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4077 }
4078
4079 /*
4080 * Guest TR.
4081 */
4082 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4083 {
4084 /*
4085 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4086 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4087 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4088 */
4089 uint16_t u16Sel = 0;
4090 uint32_t u32Limit = 0;
4091 uint64_t u64Base = 0;
4092 uint32_t u32AccessRights = 0;
4093
4094 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4095 {
4096 u16Sel = pMixedCtx->tr.Sel;
4097 u32Limit = pMixedCtx->tr.u32Limit;
4098 u64Base = pMixedCtx->tr.u64Base;
4099 u32AccessRights = pMixedCtx->tr.Attr.u;
4100 }
4101 else
4102 {
4103 Assert(pVM->hm.s.vmx.pRealModeTSS);
4104 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4105
4106 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4107 RTGCPHYS GCPhys;
4108 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4109 AssertRCReturn(rc, rc);
4110
4111 X86DESCATTR DescAttr;
4112 DescAttr.u = 0;
4113 DescAttr.n.u1Present = 1;
4114 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4115
4116 u16Sel = 0;
4117 u32Limit = HM_VTX_TSS_SIZE;
4118 u64Base = GCPhys; /* in real-mode phys = virt. */
4119 u32AccessRights = DescAttr.u;
4120 }
4121
4122 /* Validate. */
4123 Assert(!(u16Sel & RT_BIT(2)));
4124 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4125 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4126 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4127 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4128 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4129 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4130 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4131 Assert( (u32Limit & 0xfff) == 0xfff
4132 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4133 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4134 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4135
4136 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4137 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4138 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4139 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4140
4141 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4142 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4143 }
4144
4145 /*
4146 * Guest GDTR.
4147 */
4148 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4149 {
4150 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4151 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4152
4153 /* Validate. */
4154 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4155
4156 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4157 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4158 }
4159
4160 /*
4161 * Guest LDTR.
4162 */
4163 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4164 {
4165 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4166 uint32_t u32Access = 0;
4167 if (!pMixedCtx->ldtr.Attr.u)
4168 u32Access = X86DESCATTR_UNUSABLE;
4169 else
4170 u32Access = pMixedCtx->ldtr.Attr.u;
4171
4172 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4173 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4174 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4175 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4176
4177 /* Validate. */
4178 if (!(u32Access & X86DESCATTR_UNUSABLE))
4179 {
4180 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4181 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4182 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4183 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4184 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4185 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4186 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4187 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4188 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4189 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4190 }
4191
4192 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4193 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4194 }
4195
4196 /*
4197 * Guest IDTR.
4198 */
4199 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4200 {
4201 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4202 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4203
4204 /* Validate. */
4205 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4206
4207 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4208 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4209 }
4210
4211 return VINF_SUCCESS;
4212}
4213
4214
4215/**
4216 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4217 * areas. These MSRs will automatically be loaded to the host CPU on every
4218 * successful VM entry and stored from the host CPU on every successful VM exit.
4219 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4220 *
4221 * @returns VBox status code.
4222 * @param pVCpu Pointer to the VMCPU.
4223 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4224 * out-of-sync. Make sure to update the required fields
4225 * before using them.
4226 *
4227 * @remarks No-long-jump zone!!!
4228 */
4229static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4230{
4231 AssertPtr(pVCpu);
4232 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4233
4234 /*
4235 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4236 */
4237 int rc = VINF_SUCCESS;
4238 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4239 {
4240#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4241 PVM pVM = pVCpu->CTX_SUFF(pVM);
4242 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4243 uint32_t cGuestMsrs = 0;
4244
4245 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4246 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4247 * when the guest really is in 64-bit mode. */
4248 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4249 if (fSupportsLongMode)
4250 {
4251 pGuestMsr->u32Msr = MSR_K8_LSTAR;
4252 pGuestMsr->u32Reserved = 0;
4253 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4254 pGuestMsr++; cGuestMsrs++;
4255 pGuestMsr->u32Msr = MSR_K6_STAR;
4256 pGuestMsr->u32Reserved = 0;
4257 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4258 pGuestMsr++; cGuestMsrs++;
4259 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4260 pGuestMsr->u32Reserved = 0;
4261 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4262 pGuestMsr++; cGuestMsrs++;
4263 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4264 pGuestMsr->u32Reserved = 0;
4265 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4266 pGuestMsr++; cGuestMsrs++;
4267 }
4268
4269 /*
4270 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4271 * load the guest's copy always (since the MSR bitmap allows passthru unconditionally).
4272 */
4273 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
4274 {
4275 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4276 pGuestMsr->u32Reserved = 0;
4277 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4278 AssertRCReturn(rc, rc);
4279 pGuestMsr++; cGuestMsrs++;
4280 }
4281
4282 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4283 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4284 {
4285 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4286 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4287 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4288 }
4289
4290 /* Update the VCPU's copy of the guest MSR count. */
4291 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4292 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4293 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4294#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4295
4296 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4297 }
4298
4299 /*
4300 * Guest Sysenter MSRs.
4301 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4302 * VM-exits on WRMSRs for these MSRs.
4303 */
4304 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4305 {
4306 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4307 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4308 }
4309
4310 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4311 {
4312 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4313 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4314 }
4315
4316 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4317 {
4318 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4319 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4320 }
4321
4322 return rc;
4323}
4324
4325
4326/**
4327 * Loads the guest activity state into the guest-state area in the VMCS.
4328 *
4329 * @returns VBox status code.
4330 * @param pVCpu Pointer to the VMCPU.
4331 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4332 * out-of-sync. Make sure to update the required fields
4333 * before using them.
4334 *
4335 * @remarks No-long-jump zone!!!
4336 */
4337static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4338{
4339 /** @todo See if we can make use of other states, e.g.
4340 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4341 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4342 {
4343 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4344 AssertRCReturn(rc, rc);
4345
4346 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4347 }
4348 return VINF_SUCCESS;
4349}
4350
4351
4352/**
4353 * Sets up the appropriate function to run guest code.
4354 *
4355 * @returns VBox status code.
4356 * @param pVCpu Pointer to the VMCPU.
4357 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4358 * out-of-sync. Make sure to update the required fields
4359 * before using them.
4360 *
4361 * @remarks No-long-jump zone!!!
4362 */
4363static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4364{
4365 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4366 {
4367#ifndef VBOX_ENABLE_64_BITS_GUESTS
4368 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4369#endif
4370 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4371#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4372 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4373 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4374 {
4375 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4376 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4377 }
4378#else
4379 /* 64-bit host or hybrid host. */
4380 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4381#endif
4382 }
4383 else
4384 {
4385 /* Guest is not in long mode, use the 32-bit handler. */
4386#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4387 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4388 {
4389 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4390 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4391 }
4392#else
4393 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4394#endif
4395 }
4396 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4397 return VINF_SUCCESS;
4398}
4399
4400
4401/**
4402 * Wrapper for running the guest code in VT-x.
4403 *
4404 * @returns VBox strict status code.
4405 * @param pVM Pointer to the VM.
4406 * @param pVCpu Pointer to the VMCPU.
4407 * @param pCtx Pointer to the guest-CPU context.
4408 *
4409 * @remarks No-long-jump zone!!!
4410 */
4411DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4412{
4413 /*
4414 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4415 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4416 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4417 */
4418 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4419 /** @todo Add stats for resume vs launch. */
4420#ifdef VBOX_WITH_KERNEL_USING_XMM
4421 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4422#else
4423 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4424#endif
4425}
4426
4427
4428/**
4429 * Reports world-switch error and dumps some useful debug info.
4430 *
4431 * @param pVM Pointer to the VM.
4432 * @param pVCpu Pointer to the VMCPU.
4433 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4434 * @param pCtx Pointer to the guest-CPU context.
4435 * @param pVmxTransient Pointer to the VMX transient structure (only
4436 * exitReason updated).
4437 */
4438static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4439{
4440 Assert(pVM);
4441 Assert(pVCpu);
4442 Assert(pCtx);
4443 Assert(pVmxTransient);
4444 HMVMX_ASSERT_PREEMPT_SAFE();
4445
4446 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4447 switch (rcVMRun)
4448 {
4449 case VERR_VMX_INVALID_VMXON_PTR:
4450 AssertFailed();
4451 break;
4452 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4453 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4454 {
4455 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4456 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4457 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4458 AssertRC(rc);
4459
4460 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4461 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4462 Cannot do it here as we may have been long preempted. */
4463
4464#ifdef VBOX_STRICT
4465 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4466 pVmxTransient->uExitReason));
4467 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4468 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4469 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4470 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4471 else
4472 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4473 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4474 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4475
4476 /* VMX control bits. */
4477 uint32_t u32Val;
4478 uint64_t u64Val;
4479 HMVMXHCUINTREG uHCReg;
4480 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4481 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4482 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4483 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4484 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4485 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4486 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4487 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4488 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4489 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4490 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4491 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4492 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4493 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4494 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4495 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4496 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4497 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4498 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4499 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4500 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4501 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4502 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4503 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4504 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4505 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4506 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4507 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4508 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4509 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4510 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4511 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4512 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4513 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4514 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4515 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4516 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4517 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4518 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4519 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4520 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4521 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4522
4523 /* Guest bits. */
4524 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4525 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4526 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4527 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4528 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4529 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4530 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4531 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4532
4533 /* Host bits. */
4534 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4535 Log4(("Host CR0 %#RHr\n", uHCReg));
4536 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4537 Log4(("Host CR3 %#RHr\n", uHCReg));
4538 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4539 Log4(("Host CR4 %#RHr\n", uHCReg));
4540
4541 RTGDTR HostGdtr;
4542 PCX86DESCHC pDesc;
4543 ASMGetGDTR(&HostGdtr);
4544 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4545 Log4(("Host CS %#08x\n", u32Val));
4546 if (u32Val < HostGdtr.cbGdt)
4547 {
4548 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4549 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4550 }
4551
4552 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4553 Log4(("Host DS %#08x\n", u32Val));
4554 if (u32Val < HostGdtr.cbGdt)
4555 {
4556 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4557 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4558 }
4559
4560 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4561 Log4(("Host ES %#08x\n", u32Val));
4562 if (u32Val < HostGdtr.cbGdt)
4563 {
4564 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4565 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4566 }
4567
4568 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4569 Log4(("Host FS %#08x\n", u32Val));
4570 if (u32Val < HostGdtr.cbGdt)
4571 {
4572 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4573 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4574 }
4575
4576 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4577 Log4(("Host GS %#08x\n", u32Val));
4578 if (u32Val < HostGdtr.cbGdt)
4579 {
4580 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4581 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4582 }
4583
4584 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4585 Log4(("Host SS %#08x\n", u32Val));
4586 if (u32Val < HostGdtr.cbGdt)
4587 {
4588 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4589 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4590 }
4591
4592 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4593 Log4(("Host TR %#08x\n", u32Val));
4594 if (u32Val < HostGdtr.cbGdt)
4595 {
4596 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4597 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4598 }
4599
4600 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4601 Log4(("Host TR Base %#RHv\n", uHCReg));
4602 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4603 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4604 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4605 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4606 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4607 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4608 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4609 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4610 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4611 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4612 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4613 Log4(("Host RSP %#RHv\n", uHCReg));
4614 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4615 Log4(("Host RIP %#RHv\n", uHCReg));
4616# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4617 if (HMVMX_IS_64BIT_HOST_MODE())
4618 {
4619 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4620 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4621 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4622 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4623 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4624 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4625 }
4626# endif
4627#endif /* VBOX_STRICT */
4628 break;
4629 }
4630
4631 default:
4632 /* Impossible */
4633 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4634 break;
4635 }
4636 NOREF(pVM);
4637}
4638
4639
4640#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4641#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4642# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4643#endif
4644#ifdef VBOX_STRICT
4645static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4646{
4647 switch (idxField)
4648 {
4649 case VMX_VMCS_GUEST_RIP:
4650 case VMX_VMCS_GUEST_RSP:
4651 case VMX_VMCS_GUEST_SYSENTER_EIP:
4652 case VMX_VMCS_GUEST_SYSENTER_ESP:
4653 case VMX_VMCS_GUEST_GDTR_BASE:
4654 case VMX_VMCS_GUEST_IDTR_BASE:
4655 case VMX_VMCS_GUEST_CS_BASE:
4656 case VMX_VMCS_GUEST_DS_BASE:
4657 case VMX_VMCS_GUEST_ES_BASE:
4658 case VMX_VMCS_GUEST_FS_BASE:
4659 case VMX_VMCS_GUEST_GS_BASE:
4660 case VMX_VMCS_GUEST_SS_BASE:
4661 case VMX_VMCS_GUEST_LDTR_BASE:
4662 case VMX_VMCS_GUEST_TR_BASE:
4663 case VMX_VMCS_GUEST_CR3:
4664 return true;
4665 }
4666 return false;
4667}
4668
4669static bool hmR0VmxIsValidReadField(uint32_t idxField)
4670{
4671 switch (idxField)
4672 {
4673 /* Read-only fields. */
4674 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4675 return true;
4676 }
4677 /* Remaining readable fields should also be writable. */
4678 return hmR0VmxIsValidWriteField(idxField);
4679}
4680#endif /* VBOX_STRICT */
4681
4682
4683/**
4684 * Executes the specified handler in 64-bit mode.
4685 *
4686 * @returns VBox status code.
4687 * @param pVM Pointer to the VM.
4688 * @param pVCpu Pointer to the VMCPU.
4689 * @param pCtx Pointer to the guest CPU context.
4690 * @param enmOp The operation to perform.
4691 * @param cbParam Number of parameters.
4692 * @param paParam Array of 32-bit parameters.
4693 */
4694VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4695 uint32_t *paParam)
4696{
4697 int rc, rc2;
4698 PHMGLOBALCPUINFO pCpu;
4699 RTHCPHYS HCPhysCpuPage;
4700 RTCCUINTREG uOldEflags;
4701
4702 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4703 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4704 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4705 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4706
4707#ifdef VBOX_STRICT
4708 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4709 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4710
4711 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4712 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4713#endif
4714
4715 /* Disable interrupts. */
4716 uOldEflags = ASMIntDisableFlags();
4717
4718#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4719 RTCPUID idHostCpu = RTMpCpuId();
4720 CPUMR0SetLApic(pVCpu, idHostCpu);
4721#endif
4722
4723 pCpu = HMR0GetCurrentCpu();
4724 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4725
4726 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4727 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4728
4729 /* Leave VMX Root Mode. */
4730 VMXDisable();
4731
4732 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4733
4734 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4735 CPUMSetHyperEIP(pVCpu, enmOp);
4736 for (int i = (int)cbParam - 1; i >= 0; i--)
4737 CPUMPushHyper(pVCpu, paParam[i]);
4738
4739 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4740
4741 /* Call the switcher. */
4742 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4743 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4744
4745 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4746 /* Make sure the VMX instructions don't cause #UD faults. */
4747 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4748
4749 /* Re-enter VMX Root Mode */
4750 rc2 = VMXEnable(HCPhysCpuPage);
4751 if (RT_FAILURE(rc2))
4752 {
4753 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4754 ASMSetFlags(uOldEflags);
4755 return rc2;
4756 }
4757
4758 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4759 AssertRC(rc2);
4760 Assert(!(ASMGetFlags() & X86_EFL_IF));
4761 ASMSetFlags(uOldEflags);
4762 return rc;
4763}
4764
4765
4766/**
4767 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4768 * supporting 64-bit guests.
4769 *
4770 * @returns VBox status code.
4771 * @param fResume Whether to VMLAUNCH or VMRESUME.
4772 * @param pCtx Pointer to the guest-CPU context.
4773 * @param pCache Pointer to the VMCS cache.
4774 * @param pVM Pointer to the VM.
4775 * @param pVCpu Pointer to the VMCPU.
4776 */
4777DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4778{
4779 uint32_t aParam[6];
4780 PHMGLOBALCPUINFO pCpu = NULL;
4781 RTHCPHYS HCPhysCpuPage = 0;
4782 int rc = VERR_INTERNAL_ERROR_5;
4783
4784 pCpu = HMR0GetCurrentCpu();
4785 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4786
4787#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4788 pCache->uPos = 1;
4789 pCache->interPD = PGMGetInterPaeCR3(pVM);
4790 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4791#endif
4792
4793#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4794 pCache->TestIn.HCPhysCpuPage = 0;
4795 pCache->TestIn.HCPhysVmcs = 0;
4796 pCache->TestIn.pCache = 0;
4797 pCache->TestOut.HCPhysVmcs = 0;
4798 pCache->TestOut.pCache = 0;
4799 pCache->TestOut.pCtx = 0;
4800 pCache->TestOut.eflags = 0;
4801#endif
4802
4803 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4804 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4805 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4806 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4807 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4808 aParam[5] = 0;
4809
4810#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4811 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4812 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4813#endif
4814 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4815
4816#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4817 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4818 Assert(pCtx->dr[4] == 10);
4819 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4820#endif
4821
4822#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4823 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4824 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4825 pVCpu->hm.s.vmx.HCPhysVmcs));
4826 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4827 pCache->TestOut.HCPhysVmcs));
4828 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4829 pCache->TestOut.pCache));
4830 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4831 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4832 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4833 pCache->TestOut.pCtx));
4834 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4835#endif
4836 return rc;
4837}
4838
4839
4840/**
4841 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4842 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4843 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4844 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4845 *
4846 * @returns VBox status code.
4847 * @param pVM Pointer to the VM.
4848 * @param pVCpu Pointer to the VMCPU.
4849 */
4850static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4851{
4852#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4853{ \
4854 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4855 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4856 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4857 ++cReadFields; \
4858}
4859
4860 AssertPtr(pVM);
4861 AssertPtr(pVCpu);
4862 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4863 uint32_t cReadFields = 0;
4864
4865 /*
4866 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4867 * and serve to indicate exceptions to the rules.
4868 */
4869
4870 /* Guest-natural selector base fields. */
4871#if 0
4872 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4873 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4874 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4875#endif
4876 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4877 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4878 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4879 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4880 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4881 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4882 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4883 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4884 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4885 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4886 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4887 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4888#if 0
4889 /* Unused natural width guest-state fields. */
4890 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4891 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4892#endif
4893 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4894 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4895
4896 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4897#if 0
4898 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4899 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4900 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4901 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4902 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4903 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4904 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4905 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4906 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4907#endif
4908
4909 /* Natural width guest-state fields. */
4910 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4911#if 0
4912 /* Currently unused field. */
4913 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4914#endif
4915
4916 if (pVM->hm.s.fNestedPaging)
4917 {
4918 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4919 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4920 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4921 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4922 }
4923 else
4924 {
4925 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4926 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4927 }
4928
4929#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4930 return VINF_SUCCESS;
4931}
4932
4933
4934/**
4935 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4936 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4937 * darwin, running 64-bit guests).
4938 *
4939 * @returns VBox status code.
4940 * @param pVCpu Pointer to the VMCPU.
4941 * @param idxField The VMCS field encoding.
4942 * @param u64Val 16, 32 or 64 bits value.
4943 */
4944VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4945{
4946 int rc;
4947 switch (idxField)
4948 {
4949 /*
4950 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4951 */
4952 /* 64-bit Control fields. */
4953 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4954 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4955 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4956 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4957 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4958 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4959 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4960 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4961 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4962 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4963 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4964 case VMX_VMCS64_CTRL_EPTP_FULL:
4965 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4966 /* 64-bit Guest-state fields. */
4967 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4968 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4969 case VMX_VMCS64_GUEST_PAT_FULL:
4970 case VMX_VMCS64_GUEST_EFER_FULL:
4971 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4972 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4973 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4974 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4975 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4976 /* 64-bit Host-state fields. */
4977 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4978 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4979 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4980 {
4981 rc = VMXWriteVmcs32(idxField, u64Val);
4982 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4983 break;
4984 }
4985
4986 /*
4987 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4988 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4989 */
4990 /* Natural-width Guest-state fields. */
4991 case VMX_VMCS_GUEST_CR3:
4992 case VMX_VMCS_GUEST_ES_BASE:
4993 case VMX_VMCS_GUEST_CS_BASE:
4994 case VMX_VMCS_GUEST_SS_BASE:
4995 case VMX_VMCS_GUEST_DS_BASE:
4996 case VMX_VMCS_GUEST_FS_BASE:
4997 case VMX_VMCS_GUEST_GS_BASE:
4998 case VMX_VMCS_GUEST_LDTR_BASE:
4999 case VMX_VMCS_GUEST_TR_BASE:
5000 case VMX_VMCS_GUEST_GDTR_BASE:
5001 case VMX_VMCS_GUEST_IDTR_BASE:
5002 case VMX_VMCS_GUEST_RSP:
5003 case VMX_VMCS_GUEST_RIP:
5004 case VMX_VMCS_GUEST_SYSENTER_ESP:
5005 case VMX_VMCS_GUEST_SYSENTER_EIP:
5006 {
5007 if (!(u64Val >> 32))
5008 {
5009 /* If this field is 64-bit, VT-x will zero out the top bits. */
5010 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5011 }
5012 else
5013 {
5014 /* Assert that only the 32->64 switcher case should ever come here. */
5015 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5016 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5017 }
5018 break;
5019 }
5020
5021 default:
5022 {
5023 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5024 rc = VERR_INVALID_PARAMETER;
5025 break;
5026 }
5027 }
5028 AssertRCReturn(rc, rc);
5029 return rc;
5030}
5031
5032
5033/**
5034 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5035 * hosts (except darwin) for 64-bit guests.
5036 *
5037 * @param pVCpu Pointer to the VMCPU.
5038 * @param idxField The VMCS field encoding.
5039 * @param u64Val 16, 32 or 64 bits value.
5040 */
5041VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5042{
5043 AssertPtr(pVCpu);
5044 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5045
5046 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5047 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5048
5049 /* Make sure there are no duplicates. */
5050 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5051 {
5052 if (pCache->Write.aField[i] == idxField)
5053 {
5054 pCache->Write.aFieldVal[i] = u64Val;
5055 return VINF_SUCCESS;
5056 }
5057 }
5058
5059 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5060 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5061 pCache->Write.cValidEntries++;
5062 return VINF_SUCCESS;
5063}
5064
5065/* Enable later when the assembly code uses these as callbacks. */
5066#if 0
5067/*
5068 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5069 *
5070 * @param pVCpu Pointer to the VMCPU.
5071 * @param pCache Pointer to the VMCS cache.
5072 *
5073 * @remarks No-long-jump zone!!!
5074 */
5075VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5076{
5077 AssertPtr(pCache);
5078 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5079 {
5080 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5081 AssertRC(rc);
5082 }
5083 pCache->Write.cValidEntries = 0;
5084}
5085
5086
5087/**
5088 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5089 *
5090 * @param pVCpu Pointer to the VMCPU.
5091 * @param pCache Pointer to the VMCS cache.
5092 *
5093 * @remarks No-long-jump zone!!!
5094 */
5095VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5096{
5097 AssertPtr(pCache);
5098 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5099 {
5100 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5101 AssertRC(rc);
5102 }
5103}
5104#endif
5105#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5106
5107
5108/**
5109 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5110 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5111 * timer.
5112 *
5113 * @returns VBox status code.
5114 * @param pVCpu Pointer to the VMCPU.
5115 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5116 * out-of-sync. Make sure to update the required fields
5117 * before using them.
5118 * @remarks No-long-jump zone!!!
5119 */
5120static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5121{
5122 int rc = VERR_INTERNAL_ERROR_5;
5123 bool fOffsettedTsc = false;
5124 PVM pVM = pVCpu->CTX_SUFF(pVM);
5125 if (pVM->hm.s.vmx.fUsePreemptTimer)
5126 {
5127 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5128
5129 /* Make sure the returned values have sane upper and lower boundaries. */
5130 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5131 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5132 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5133 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5134
5135 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5136 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5137 }
5138 else
5139 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5140
5141 if (fOffsettedTsc)
5142 {
5143 uint64_t u64CurTSC = ASMReadTSC();
5144 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5145 {
5146 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5147 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5148
5149 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5150 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5151 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5152 }
5153 else
5154 {
5155 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5156 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5157 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5158 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5159 }
5160 }
5161 else
5162 {
5163 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5164 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5165 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5166 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5167 }
5168}
5169
5170
5171/**
5172 * Determines if an exception is a contributory exception. Contributory
5173 * exceptions are ones which can cause double-faults. Page-fault is
5174 * intentionally not included here as it's a conditional contributory exception.
5175 *
5176 * @returns true if the exception is contributory, false otherwise.
5177 * @param uVector The exception vector.
5178 */
5179DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5180{
5181 switch (uVector)
5182 {
5183 case X86_XCPT_GP:
5184 case X86_XCPT_SS:
5185 case X86_XCPT_NP:
5186 case X86_XCPT_TS:
5187 case X86_XCPT_DE:
5188 return true;
5189 default:
5190 break;
5191 }
5192 return false;
5193}
5194
5195
5196/**
5197 * Sets an event as a pending event to be injected into the guest.
5198 *
5199 * @param pVCpu Pointer to the VMCPU.
5200 * @param u32IntInfo The VM-entry interruption-information field.
5201 * @param cbInstr The VM-entry instruction length in bytes (for software
5202 * interrupts, exceptions and privileged software
5203 * exceptions).
5204 * @param u32ErrCode The VM-entry exception error code.
5205 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5206 * page-fault.
5207 *
5208 * @remarks Statistics counter assumes this is a guest event being injected or
5209 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5210 * always incremented.
5211 */
5212DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5213 RTGCUINTPTR GCPtrFaultAddress)
5214{
5215 Assert(!pVCpu->hm.s.Event.fPending);
5216 pVCpu->hm.s.Event.fPending = true;
5217 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5218 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5219 pVCpu->hm.s.Event.cbInstr = cbInstr;
5220 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5221
5222 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5223}
5224
5225
5226/**
5227 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5228 *
5229 * @param pVCpu Pointer to the VMCPU.
5230 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5231 * out-of-sync. Make sure to update the required fields
5232 * before using them.
5233 */
5234DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5235{
5236 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5237 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5238 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5239 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5240}
5241
5242
5243/**
5244 * Handle a condition that occurred while delivering an event through the guest
5245 * IDT.
5246 *
5247 * @returns VBox status code (informational error codes included).
5248 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5249 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5250 * continue execution of the guest which will delivery the #DF.
5251 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5252 *
5253 * @param pVCpu Pointer to the VMCPU.
5254 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5255 * out-of-sync. Make sure to update the required fields
5256 * before using them.
5257 * @param pVmxTransient Pointer to the VMX transient structure.
5258 *
5259 * @remarks No-long-jump zone!!!
5260 */
5261static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5262{
5263 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5264 AssertRCReturn(rc, rc);
5265 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5266 {
5267 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5268 AssertRCReturn(rc, rc);
5269
5270 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5271 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5272 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5273
5274 typedef enum
5275 {
5276 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5277 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5278 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5279 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5280 } VMXREFLECTXCPT;
5281
5282 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5283 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5284 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5285 {
5286 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5287 {
5288 enmReflect = VMXREFLECTXCPT_XCPT;
5289#ifdef VBOX_STRICT
5290 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5291 && uExitVector == X86_XCPT_PF)
5292 {
5293 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5294 }
5295#endif
5296 if ( uExitVector == X86_XCPT_PF
5297 && uIdtVector == X86_XCPT_PF)
5298 {
5299 pVmxTransient->fVectoringPF = true;
5300 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5301 }
5302 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5303 && hmR0VmxIsContributoryXcpt(uExitVector)
5304 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5305 || uIdtVector == X86_XCPT_PF))
5306 {
5307 enmReflect = VMXREFLECTXCPT_DF;
5308 }
5309 else if (uIdtVector == X86_XCPT_DF)
5310 enmReflect = VMXREFLECTXCPT_TF;
5311 }
5312 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5313 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5314 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5315 {
5316 /*
5317 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5318 * (whatever they are) as they reoccur when restarting the instruction.
5319 */
5320 enmReflect = VMXREFLECTXCPT_XCPT;
5321 }
5322 }
5323 else
5324 {
5325 /*
5326 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5327 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5328 * original exception to the guest after handling the VM-exit.
5329 */
5330 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5331 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5332 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5333 {
5334 enmReflect = VMXREFLECTXCPT_XCPT;
5335 }
5336 }
5337
5338 switch (enmReflect)
5339 {
5340 case VMXREFLECTXCPT_XCPT:
5341 {
5342 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5343 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5344 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5345
5346 uint32_t u32ErrCode = 0;
5347 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5348 {
5349 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5350 AssertRCReturn(rc, rc);
5351 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5352 }
5353
5354 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5355 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5356 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5357 rc = VINF_SUCCESS;
5358 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5359 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5360
5361 break;
5362 }
5363
5364 case VMXREFLECTXCPT_DF:
5365 {
5366 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5367 rc = VINF_HM_DOUBLE_FAULT;
5368 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5369 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5370
5371 break;
5372 }
5373
5374 case VMXREFLECTXCPT_TF:
5375 {
5376 rc = VINF_EM_RESET;
5377 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5378 uExitVector));
5379 break;
5380 }
5381
5382 default:
5383 Assert(rc == VINF_SUCCESS);
5384 break;
5385 }
5386 }
5387 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5388 return rc;
5389}
5390
5391
5392/**
5393 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5394 *
5395 * @returns VBox status code.
5396 * @param pVCpu Pointer to the VMCPU.
5397 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5398 * out-of-sync. Make sure to update the required fields
5399 * before using them.
5400 *
5401 * @remarks No-long-jump zone!!!
5402 */
5403static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5404{
5405 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5406 {
5407 uint32_t uVal = 0;
5408 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5409 AssertRCReturn(rc, rc);
5410
5411 uint32_t uShadow = 0;
5412 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5413 AssertRCReturn(rc, rc);
5414
5415 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5416 CPUMSetGuestCR0(pVCpu, uVal);
5417 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5418 }
5419 return VINF_SUCCESS;
5420}
5421
5422
5423/**
5424 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5425 *
5426 * @returns VBox status code.
5427 * @param pVCpu Pointer to the VMCPU.
5428 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5429 * out-of-sync. Make sure to update the required fields
5430 * before using them.
5431 *
5432 * @remarks No-long-jump zone!!!
5433 */
5434static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5435{
5436 int rc = VINF_SUCCESS;
5437 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5438 {
5439 uint32_t uVal = 0;
5440 uint32_t uShadow = 0;
5441 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5442 AssertRCReturn(rc, rc);
5443 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5444 AssertRCReturn(rc, rc);
5445
5446 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5447 CPUMSetGuestCR4(pVCpu, uVal);
5448 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5449 }
5450 return rc;
5451}
5452
5453
5454/**
5455 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5456 *
5457 * @returns VBox status code.
5458 * @param pVCpu Pointer to the VMCPU.
5459 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5460 * out-of-sync. Make sure to update the required fields
5461 * before using them.
5462 *
5463 * @remarks No-long-jump zone!!!
5464 */
5465static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5466{
5467 int rc = VINF_SUCCESS;
5468 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5469 {
5470 uint64_t u64Val = 0;
5471 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5472 AssertRCReturn(rc, rc);
5473
5474 pMixedCtx->rip = u64Val;
5475 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5476 }
5477 return rc;
5478}
5479
5480
5481/**
5482 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5483 *
5484 * @returns VBox status code.
5485 * @param pVCpu Pointer to the VMCPU.
5486 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5487 * out-of-sync. Make sure to update the required fields
5488 * before using them.
5489 *
5490 * @remarks No-long-jump zone!!!
5491 */
5492static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5493{
5494 int rc = VINF_SUCCESS;
5495 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5496 {
5497 uint64_t u64Val = 0;
5498 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5499 AssertRCReturn(rc, rc);
5500
5501 pMixedCtx->rsp = u64Val;
5502 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5503 }
5504 return rc;
5505}
5506
5507
5508/**
5509 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5510 *
5511 * @returns VBox status code.
5512 * @param pVCpu Pointer to the VMCPU.
5513 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5514 * out-of-sync. Make sure to update the required fields
5515 * before using them.
5516 *
5517 * @remarks No-long-jump zone!!!
5518 */
5519static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5520{
5521 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5522 {
5523 uint32_t uVal = 0;
5524 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5525 AssertRCReturn(rc, rc);
5526
5527 pMixedCtx->eflags.u32 = uVal;
5528 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5529 {
5530 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5531 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5532
5533 pMixedCtx->eflags.Bits.u1VM = 0;
5534 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5535 }
5536
5537 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5538 }
5539 return VINF_SUCCESS;
5540}
5541
5542
5543/**
5544 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5545 * guest-CPU context.
5546 */
5547DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5548{
5549 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5550 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5551 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5552 return rc;
5553}
5554
5555
5556/**
5557 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5558 * from the guest-state area in the VMCS.
5559 *
5560 * @param pVCpu Pointer to the VMCPU.
5561 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5562 * out-of-sync. Make sure to update the required fields
5563 * before using them.
5564 *
5565 * @remarks No-long-jump zone!!!
5566 */
5567static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5568{
5569 uint32_t uIntrState = 0;
5570 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5571 AssertRC(rc);
5572
5573 if (!uIntrState)
5574 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5575 else
5576 {
5577 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5578 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5579 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5580 AssertRC(rc);
5581 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5582 AssertRC(rc);
5583
5584 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5585 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5586 }
5587}
5588
5589
5590/**
5591 * Saves the guest's activity state.
5592 *
5593 * @returns VBox status code.
5594 * @param pVCpu Pointer to the VMCPU.
5595 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5596 * out-of-sync. Make sure to update the required fields
5597 * before using them.
5598 *
5599 * @remarks No-long-jump zone!!!
5600 */
5601static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5602{
5603 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5604 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5605 return VINF_SUCCESS;
5606}
5607
5608
5609/**
5610 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5611 * the current VMCS into the guest-CPU context.
5612 *
5613 * @returns VBox status code.
5614 * @param pVCpu Pointer to the VMCPU.
5615 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5616 * out-of-sync. Make sure to update the required fields
5617 * before using them.
5618 *
5619 * @remarks No-long-jump zone!!!
5620 */
5621static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5622{
5623 int rc = VINF_SUCCESS;
5624 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5625 {
5626 uint32_t u32Val = 0;
5627 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5628 pMixedCtx->SysEnter.cs = u32Val;
5629 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5630 }
5631
5632 uint64_t u64Val = 0;
5633 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5634 {
5635 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5636 pMixedCtx->SysEnter.eip = u64Val;
5637 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5638 }
5639 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5640 {
5641 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5642 pMixedCtx->SysEnter.esp = u64Val;
5643 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5644 }
5645 return rc;
5646}
5647
5648
5649/**
5650 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5651 * context.
5652 *
5653 * @returns VBox status code.
5654 * @param pVCpu Pointer to the VMCPU.
5655 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5656 * out-of-sync. Make sure to update the required fields
5657 * before using them.
5658 *
5659 * @remarks No-long-jump zone!!!
5660 */
5661static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5662{
5663 int rc = VINF_SUCCESS;
5664 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5665 {
5666 uint64_t u64Val = 0;
5667 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5668 pMixedCtx->fs.u64Base = u64Val;
5669 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5670 }
5671 return rc;
5672}
5673
5674
5675/**
5676 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5677 * context.
5678 *
5679 * @returns VBox status code.
5680 * @param pVCpu Pointer to the VMCPU.
5681 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5682 * out-of-sync. Make sure to update the required fields
5683 * before using them.
5684 *
5685 * @remarks No-long-jump zone!!!
5686 */
5687static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5688{
5689 int rc = VINF_SUCCESS;
5690 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5691 {
5692 uint64_t u64Val = 0;
5693 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5694 pMixedCtx->gs.u64Base = u64Val;
5695 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5696 }
5697 return rc;
5698}
5699
5700
5701/**
5702 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5703 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5704 * and TSC_AUX.
5705 *
5706 * @returns VBox status code.
5707 * @param pVCpu Pointer to the VMCPU.
5708 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5709 * out-of-sync. Make sure to update the required fields
5710 * before using them.
5711 *
5712 * @remarks No-long-jump zone!!!
5713 */
5714static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5715{
5716 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5717 return VINF_SUCCESS;
5718
5719#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5720 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5721 {
5722 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5723 pMsr += i;
5724 switch (pMsr->u32Msr)
5725 {
5726 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5727 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5728 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5729 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5730 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5731 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5732 default:
5733 {
5734 AssertFailed();
5735 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5736 }
5737 }
5738 }
5739#endif
5740
5741 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5742 return VINF_SUCCESS;
5743}
5744
5745
5746/**
5747 * Saves the guest control registers from the current VMCS into the guest-CPU
5748 * context.
5749 *
5750 * @returns VBox status code.
5751 * @param pVCpu Pointer to the VMCPU.
5752 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5753 * out-of-sync. Make sure to update the required fields
5754 * before using them.
5755 *
5756 * @remarks No-long-jump zone!!!
5757 */
5758static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5759{
5760 /* Guest CR0. Guest FPU. */
5761 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5762 AssertRCReturn(rc, rc);
5763
5764 /* Guest CR4. */
5765 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5766 AssertRCReturn(rc, rc);
5767
5768 /* Guest CR2 - updated always during the world-switch or in #PF. */
5769 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5770 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5771 {
5772 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5773 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5774
5775 PVM pVM = pVCpu->CTX_SUFF(pVM);
5776 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5777 || ( pVM->hm.s.fNestedPaging
5778 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5779 {
5780 uint64_t u64Val = 0;
5781 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5782 if (pMixedCtx->cr3 != u64Val)
5783 {
5784 CPUMSetGuestCR3(pVCpu, u64Val);
5785 if (VMMRZCallRing3IsEnabled(pVCpu))
5786 {
5787 PGMUpdateCR3(pVCpu, u64Val);
5788 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5789 }
5790 else
5791 {
5792 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5793 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5794 }
5795 }
5796
5797 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5798 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5799 {
5800 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5801 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5802 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5803 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5804
5805 if (VMMRZCallRing3IsEnabled(pVCpu))
5806 {
5807 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5808 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5809 }
5810 else
5811 {
5812 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5813 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5814 }
5815 }
5816 }
5817
5818 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5819 }
5820
5821 /*
5822 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5823 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5824 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5825 *
5826 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5827 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5828 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5829 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5830 *
5831 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5832 */
5833 if (VMMRZCallRing3IsEnabled(pVCpu))
5834 {
5835 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5836 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5837
5838 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5839 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5840
5841 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5842 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5843 }
5844
5845 return rc;
5846}
5847
5848
5849/**
5850 * Reads a guest segment register from the current VMCS into the guest-CPU
5851 * context.
5852 *
5853 * @returns VBox status code.
5854 * @param pVCpu Pointer to the VMCPU.
5855 * @param idxSel Index of the selector in the VMCS.
5856 * @param idxLimit Index of the segment limit in the VMCS.
5857 * @param idxBase Index of the segment base in the VMCS.
5858 * @param idxAccess Index of the access rights of the segment in the VMCS.
5859 * @param pSelReg Pointer to the segment selector.
5860 *
5861 * @remarks No-long-jump zone!!!
5862 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5863 * macro as that takes care of whether to read from the VMCS cache or
5864 * not.
5865 */
5866DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5867 PCPUMSELREG pSelReg)
5868{
5869 uint32_t u32Val = 0;
5870 int rc = VMXReadVmcs32(idxSel, &u32Val);
5871 AssertRCReturn(rc, rc);
5872 pSelReg->Sel = (uint16_t)u32Val;
5873 pSelReg->ValidSel = (uint16_t)u32Val;
5874 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5875
5876 rc = VMXReadVmcs32(idxLimit, &u32Val);
5877 AssertRCReturn(rc, rc);
5878 pSelReg->u32Limit = u32Val;
5879
5880 uint64_t u64Val = 0;
5881 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5882 AssertRCReturn(rc, rc);
5883 pSelReg->u64Base = u64Val;
5884
5885 rc = VMXReadVmcs32(idxAccess, &u32Val);
5886 AssertRCReturn(rc, rc);
5887 pSelReg->Attr.u = u32Val;
5888
5889 /*
5890 * If VT-x marks the segment as unusable, most other bits remain undefined:
5891 * - For CS the L, D and G bits have meaning.
5892 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5893 * - For the remaining data segments no bits are defined.
5894 *
5895 * The present bit and the unusable bit has been observed to be set at the
5896 * same time (the selector was supposed to invalid as we started executing
5897 * a V8086 interrupt in ring-0).
5898 *
5899 * What should be important for the rest of the VBox code that the P bit is
5900 * cleared. Some of the other VBox code recognizes the unusable bit, but
5901 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5902 * safe side here, we'll strip off P and other bits we don't care about. If
5903 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5904 *
5905 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5906 */
5907 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5908 {
5909 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5910
5911 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5912 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5913 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5914
5915 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5916#ifdef DEBUG_bird
5917 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5918 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5919 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5920#endif
5921 }
5922 return VINF_SUCCESS;
5923}
5924
5925
5926#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5927# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5928 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5929 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5930#else
5931# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5932 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5933 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5934#endif
5935
5936
5937/**
5938 * Saves the guest segment registers from the current VMCS into the guest-CPU
5939 * context.
5940 *
5941 * @returns VBox status code.
5942 * @param pVCpu Pointer to the VMCPU.
5943 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5944 * out-of-sync. Make sure to update the required fields
5945 * before using them.
5946 *
5947 * @remarks No-long-jump zone!!!
5948 */
5949static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5950{
5951 /* Guest segment registers. */
5952 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5953 {
5954 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5955 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5956 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5957 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5958 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5959 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5960 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5961
5962 /* Restore segment attributes for real-on-v86 mode hack. */
5963 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5964 {
5965 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5966 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5967 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5968 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5969 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5970 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5971 }
5972 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5973 }
5974
5975 return VINF_SUCCESS;
5976}
5977
5978
5979/**
5980 * Saves the guest descriptor table registers and task register from the current
5981 * VMCS into the guest-CPU context.
5982 *
5983 * @returns VBox status code.
5984 * @param pVCpu Pointer to the VMCPU.
5985 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5986 * out-of-sync. Make sure to update the required fields
5987 * before using them.
5988 *
5989 * @remarks No-long-jump zone!!!
5990 */
5991static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5992{
5993 int rc = VINF_SUCCESS;
5994
5995 /* Guest LDTR. */
5996 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5997 {
5998 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5999 AssertRCReturn(rc, rc);
6000 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
6001 }
6002
6003 /* Guest GDTR. */
6004 uint64_t u64Val = 0;
6005 uint32_t u32Val = 0;
6006 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6007 {
6008 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6009 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6010 pMixedCtx->gdtr.pGdt = u64Val;
6011 pMixedCtx->gdtr.cbGdt = u32Val;
6012 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
6013 }
6014
6015 /* Guest IDTR. */
6016 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6017 {
6018 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6019 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6020 pMixedCtx->idtr.pIdt = u64Val;
6021 pMixedCtx->idtr.cbIdt = u32Val;
6022 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
6023 }
6024
6025 /* Guest TR. */
6026 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6027 {
6028 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6029 AssertRCReturn(rc, rc);
6030
6031 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6032 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6033 {
6034 rc = VMXLOCAL_READ_SEG(TR, tr);
6035 AssertRCReturn(rc, rc);
6036 }
6037 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
6038 }
6039 return rc;
6040}
6041
6042#undef VMXLOCAL_READ_SEG
6043
6044
6045/**
6046 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6047 * context.
6048 *
6049 * @returns VBox status code.
6050 * @param pVCpu Pointer to the VMCPU.
6051 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6052 * out-of-sync. Make sure to update the required fields
6053 * before using them.
6054 *
6055 * @remarks No-long-jump zone!!!
6056 */
6057static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6058{
6059 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6060 {
6061 if (!pVCpu->hm.s.fUsingHyperDR7)
6062 {
6063 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6064 uint32_t u32Val;
6065 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6066 pMixedCtx->dr[7] = u32Val;
6067 }
6068
6069 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
6070 }
6071 return VINF_SUCCESS;
6072}
6073
6074
6075/**
6076 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6077 *
6078 * @returns VBox status code.
6079 * @param pVCpu Pointer to the VMCPU.
6080 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6081 * out-of-sync. Make sure to update the required fields
6082 * before using them.
6083 *
6084 * @remarks No-long-jump zone!!!
6085 */
6086static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6087{
6088 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6089 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
6090 return VINF_SUCCESS;
6091}
6092
6093
6094/**
6095 * Saves the entire guest state from the currently active VMCS into the
6096 * guest-CPU context. This essentially VMREADs all guest-data.
6097 *
6098 * @returns VBox status code.
6099 * @param pVCpu Pointer to the VMCPU.
6100 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6101 * out-of-sync. Make sure to update the required fields
6102 * before using them.
6103 */
6104static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6105{
6106 Assert(pVCpu);
6107 Assert(pMixedCtx);
6108
6109 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6110 return VINF_SUCCESS;
6111
6112 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6113 again on the ring-3 callback path, there is no real need to. */
6114 if (VMMRZCallRing3IsEnabled(pVCpu))
6115 VMMR0LogFlushDisable(pVCpu);
6116 else
6117 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6118 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6119
6120 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6121 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6122
6123 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6124 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6125
6126 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6127 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6128
6129 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6130 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6131
6132 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6133 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6134
6135 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6136 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6137
6138 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
6139 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6140
6141 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
6142 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6143
6144 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6145 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6146
6147 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6148 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6149
6150 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6151 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6152
6153 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6154 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6155
6156 if (VMMRZCallRing3IsEnabled(pVCpu))
6157 VMMR0LogFlushEnable(pVCpu);
6158
6159 return rc;
6160}
6161
6162
6163/**
6164 * Check per-VM and per-VCPU force flag actions that require us to go back to
6165 * ring-3 for one reason or another.
6166 *
6167 * @returns VBox status code (information status code included).
6168 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6169 * ring-3.
6170 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6171 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6172 * interrupts)
6173 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6174 * all EMTs to be in ring-3.
6175 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6176 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6177 * to the EM loop.
6178 *
6179 * @param pVM Pointer to the VM.
6180 * @param pVCpu Pointer to the VMCPU.
6181 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6182 * out-of-sync. Make sure to update the required fields
6183 * before using them.
6184 */
6185static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6186{
6187 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6188
6189 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6190 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6191 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6192 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6193 {
6194 /* We need the control registers now, make sure the guest-CPU context is updated. */
6195 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6196 AssertRCReturn(rc3, rc3);
6197
6198 /* Pending HM CR3 sync. */
6199 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6200 {
6201 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6202 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6203 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6204 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6205 }
6206
6207 /* Pending HM PAE PDPEs. */
6208 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6209 {
6210 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6211 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6212 }
6213
6214 /* Pending PGM C3 sync. */
6215 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6216 {
6217 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6218 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6219 if (rc2 != VINF_SUCCESS)
6220 {
6221 AssertRC(rc2);
6222 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6223 return rc2;
6224 }
6225 }
6226
6227 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6228 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6229 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6230 {
6231 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6232 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6233 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6234 return rc2;
6235 }
6236
6237 /* Pending VM request packets, such as hardware interrupts. */
6238 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6239 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6240 {
6241 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6242 return VINF_EM_PENDING_REQUEST;
6243 }
6244
6245 /* Pending PGM pool flushes. */
6246 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6247 {
6248 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6249 return VINF_PGM_POOL_FLUSH_PENDING;
6250 }
6251
6252 /* Pending DMA requests. */
6253 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6254 {
6255 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6256 return VINF_EM_RAW_TO_R3;
6257 }
6258 }
6259
6260 return VINF_SUCCESS;
6261}
6262
6263
6264/**
6265 * Converts any TRPM trap into a pending HM event. This is typically used when
6266 * entering from ring-3 (not longjmp returns).
6267 *
6268 * @param pVCpu Pointer to the VMCPU.
6269 */
6270static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6271{
6272 Assert(TRPMHasTrap(pVCpu));
6273 Assert(!pVCpu->hm.s.Event.fPending);
6274
6275 uint8_t uVector;
6276 TRPMEVENT enmTrpmEvent;
6277 RTGCUINT uErrCode;
6278 RTGCUINTPTR GCPtrFaultAddress;
6279 uint8_t cbInstr;
6280
6281 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6282 AssertRC(rc);
6283
6284 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6285 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6286 if (enmTrpmEvent == TRPM_TRAP)
6287 {
6288 switch (uVector)
6289 {
6290 case X86_XCPT_BP:
6291 case X86_XCPT_OF:
6292 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6293 break;
6294
6295 case X86_XCPT_PF:
6296 case X86_XCPT_DF:
6297 case X86_XCPT_TS:
6298 case X86_XCPT_NP:
6299 case X86_XCPT_SS:
6300 case X86_XCPT_GP:
6301 case X86_XCPT_AC:
6302 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6303 /* no break! */
6304 default:
6305 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6306 break;
6307 }
6308 }
6309 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6310 {
6311 if (uVector == X86_XCPT_NMI)
6312 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6313 else
6314 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6315 }
6316 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6317 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6318 else
6319 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6320
6321 rc = TRPMResetTrap(pVCpu);
6322 AssertRC(rc);
6323 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6324 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6325
6326 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6327 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6328}
6329
6330
6331/**
6332 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6333 * VT-x to execute any instruction.
6334 *
6335 * @param pvCpu Pointer to the VMCPU.
6336 */
6337static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6338{
6339 Assert(pVCpu->hm.s.Event.fPending);
6340
6341 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6342 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6343 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6344 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6345
6346 /* If a trap was already pending, we did something wrong! */
6347 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6348
6349 TRPMEVENT enmTrapType;
6350 switch (uVectorType)
6351 {
6352 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6353 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6354 enmTrapType = TRPM_HARDWARE_INT;
6355 break;
6356
6357 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6358 enmTrapType = TRPM_SOFTWARE_INT;
6359 break;
6360
6361 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6362 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6363 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6364 enmTrapType = TRPM_TRAP;
6365 break;
6366
6367 default:
6368 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6369 enmTrapType = TRPM_32BIT_HACK;
6370 break;
6371 }
6372
6373 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6374
6375 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6376 AssertRC(rc);
6377
6378 if (fErrorCodeValid)
6379 TRPMSetErrorCode(pVCpu, uErrorCode);
6380
6381 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6382 && uVector == X86_XCPT_PF)
6383 {
6384 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6385 }
6386 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6387 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6388 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6389 {
6390 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6391 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6392 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6393 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6394 }
6395 pVCpu->hm.s.Event.fPending = false;
6396}
6397
6398
6399/**
6400 * Does the necessary state syncing before returning to ring-3 for any reason
6401 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6402 *
6403 * @returns VBox status code.
6404 * @param pVM Pointer to the VM.
6405 * @param pVCpu Pointer to the VMCPU.
6406 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6407 * be out-of-sync. Make sure to update the required
6408 * fields before using them.
6409 * @param fSaveGuestState Whether to save the guest state or not.
6410 *
6411 * @remarks If you modify code here, make sure to check whether
6412 * hmR0VmxCallRing3Callback() needs to be updated too.
6413 * @remarks No-long-jmp zone!!!
6414 */
6415static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6416{
6417 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6418 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6419
6420 RTCPUID idCpu = RTMpCpuId();
6421 Log4Func(("HostCpuId=%u\n", idCpu));
6422
6423 /* Save the guest state if necessary. */
6424 if ( fSaveGuestState
6425 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6426 {
6427 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6428 AssertRCReturn(rc, rc);
6429 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6430 }
6431
6432 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6433 if (CPUMIsGuestFPUStateActive(pVCpu))
6434 {
6435 /* We shouldn't reload CR0 without saving it first. */
6436 if (!fSaveGuestState)
6437 {
6438 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6439 AssertRCReturn(rc, rc);
6440 }
6441 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6442 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6443 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6444 }
6445
6446 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6447#ifdef VBOX_STRICT
6448 if (CPUMIsHyperDebugStateActive(pVCpu))
6449 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6450#endif
6451 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6452 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6453 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6454 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6455
6456#if HC_ARCH_BITS == 64
6457 /* Restore host-state bits that VT-x only restores partially. */
6458 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6459 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6460 {
6461 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6462 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6463 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6464 }
6465#endif
6466
6467 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6468 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6469 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6470 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6471 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6472 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6473 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6474 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6475
6476 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6477
6478 /** @todo This kinda defeats the purpose of having preemption hooks.
6479 * The problem is, deregistering the hooks should be moved to a place that
6480 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6481 * context.
6482 */
6483 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6484 {
6485 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6486 AssertRCReturn(rc, rc);
6487
6488 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6489 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6490 }
6491 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6492 NOREF(idCpu);
6493
6494 return VINF_SUCCESS;
6495}
6496
6497
6498/**
6499 * Leaves the VT-x session.
6500 *
6501 * @returns VBox status code.
6502 * @param pVM Pointer to the VM.
6503 * @param pVCpu Pointer to the VMCPU.
6504 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6505 * out-of-sync. Make sure to update the required fields
6506 * before using them.
6507 *
6508 * @remarks No-long-jmp zone!!!
6509 */
6510DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6511{
6512 HM_DISABLE_PREEMPT_IF_NEEDED();
6513 HMVMX_ASSERT_CPU_SAFE();
6514 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6515 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6516
6517 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6518 and done this from the VMXR0ThreadCtxCallback(). */
6519 if (!pVCpu->hm.s.fLeaveDone)
6520 {
6521 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6522 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6523 pVCpu->hm.s.fLeaveDone = true;
6524 }
6525
6526 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6527 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6528 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6529 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6530 VMMR0ThreadCtxHooksDeregister(pVCpu);
6531
6532 /* Leave HM context. This takes care of local init (term). */
6533 int rc = HMR0LeaveCpu(pVCpu);
6534
6535 HM_RESTORE_PREEMPT_IF_NEEDED();
6536
6537 return rc;
6538}
6539
6540
6541/**
6542 * Does the necessary state syncing before doing a longjmp to ring-3.
6543 *
6544 * @returns VBox status code.
6545 * @param pVM Pointer to the VM.
6546 * @param pVCpu Pointer to the VMCPU.
6547 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6548 * out-of-sync. Make sure to update the required fields
6549 * before using them.
6550 *
6551 * @remarks No-long-jmp zone!!!
6552 */
6553DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6554{
6555 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6556}
6557
6558
6559/**
6560 * Take necessary actions before going back to ring-3.
6561 *
6562 * An action requires us to go back to ring-3. This function does the necessary
6563 * steps before we can safely return to ring-3. This is not the same as longjmps
6564 * to ring-3, this is voluntary and prepares the guest so it may continue
6565 * executing outside HM (recompiler/IEM).
6566 *
6567 * @returns VBox status code.
6568 * @param pVM Pointer to the VM.
6569 * @param pVCpu Pointer to the VMCPU.
6570 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6571 * out-of-sync. Make sure to update the required fields
6572 * before using them.
6573 * @param rcExit The reason for exiting to ring-3. Can be
6574 * VINF_VMM_UNKNOWN_RING3_CALL.
6575 */
6576static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6577{
6578 Assert(pVM);
6579 Assert(pVCpu);
6580 Assert(pMixedCtx);
6581 HMVMX_ASSERT_PREEMPT_SAFE();
6582
6583 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6584 {
6585 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6586 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6587 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6588 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6589 }
6590
6591 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6592 VMMRZCallRing3Disable(pVCpu);
6593 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6594
6595 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6596 if (pVCpu->hm.s.Event.fPending)
6597 {
6598 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6599 Assert(!pVCpu->hm.s.Event.fPending);
6600 }
6601
6602 /* Save guest state and restore host state bits. */
6603 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6604 AssertRCReturn(rc, rc);
6605 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6606
6607 /* Sync recompiler state. */
6608 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6609 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6610 | CPUM_CHANGED_LDTR
6611 | CPUM_CHANGED_GDTR
6612 | CPUM_CHANGED_IDTR
6613 | CPUM_CHANGED_TR
6614 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6615 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6616 if ( pVM->hm.s.fNestedPaging
6617 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6618 {
6619 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6620 }
6621
6622 Assert(!pVCpu->hm.s.fClearTrapFlag);
6623
6624 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6625 if (rcExit != VINF_EM_RAW_INTERRUPT)
6626 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6627
6628 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6629
6630 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6631 VMMRZCallRing3RemoveNotification(pVCpu);
6632 VMMRZCallRing3Enable(pVCpu);
6633
6634 return rc;
6635}
6636
6637
6638/**
6639 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6640 * longjump to ring-3 and possibly get preempted.
6641 *
6642 * @returns VBox status code.
6643 * @param pVCpu Pointer to the VMCPU.
6644 * @param enmOperation The operation causing the ring-3 longjump.
6645 * @param pvUser Opaque pointer to the guest-CPU context. The data
6646 * may be out-of-sync. Make sure to update the required
6647 * fields before using them.
6648 * @remarks If you modify code here, make sure to check whether
6649 * hmR0VmxLeave() needs to be updated too.
6650 */
6651DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6652{
6653 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6654 {
6655 VMMRZCallRing3RemoveNotification(pVCpu);
6656 HM_DISABLE_PREEMPT_IF_NEEDED();
6657
6658 /* If anything here asserts or fails, good luck. */
6659 if (CPUMIsGuestFPUStateActive(pVCpu))
6660 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6661
6662 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6663
6664#if HC_ARCH_BITS == 64
6665 /* Restore host-state bits that VT-x only restores partially. */
6666 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6667 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6668 {
6669 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6670 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6671 }
6672#endif
6673 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6674 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6675 {
6676 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6677 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6678 }
6679
6680 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6681 VMMR0ThreadCtxHooksDeregister(pVCpu);
6682
6683 HMR0LeaveCpu(pVCpu);
6684 HM_RESTORE_PREEMPT_IF_NEEDED();
6685 return VINF_SUCCESS;
6686 }
6687
6688 Assert(pVCpu);
6689 Assert(pvUser);
6690 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6691 HMVMX_ASSERT_PREEMPT_SAFE();
6692
6693 VMMRZCallRing3Disable(pVCpu);
6694 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6695
6696 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6697 enmOperation));
6698
6699 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6700 AssertRCReturn(rc, rc);
6701
6702 VMMRZCallRing3Enable(pVCpu);
6703 return VINF_SUCCESS;
6704}
6705
6706
6707/**
6708 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6709 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6710 *
6711 * @param pVCpu Pointer to the VMCPU.
6712 */
6713DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6714{
6715 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6716 {
6717 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6718 {
6719 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6720 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6721 AssertRC(rc);
6722 }
6723 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6724}
6725
6726
6727/**
6728 * Evaluates the event to be delivered to the guest and sets it as the pending
6729 * event.
6730 *
6731 * @param pVCpu Pointer to the VMCPU.
6732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6733 * out-of-sync. Make sure to update the required fields
6734 * before using them.
6735 */
6736static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6737{
6738 Assert(!pVCpu->hm.s.Event.fPending);
6739
6740 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6741 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6742 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6743 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6744
6745 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6746 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6747 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6748 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6749 Assert(!TRPMHasTrap(pVCpu));
6750
6751 /** @todo SMI. SMIs take priority over NMIs. */
6752 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6753 {
6754 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6755 if ( !fBlockMovSS
6756 && !fBlockSti)
6757 {
6758 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6759 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6760 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6761 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6762
6763 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6764 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6765 }
6766 else
6767 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6768 }
6769 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6770 && !pVCpu->hm.s.fSingleInstruction)
6771 {
6772 /*
6773 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6774 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6775 * evaluated here and not set as pending, solely based on the force-flags.
6776 */
6777 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6778 AssertRC(rc);
6779 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6780 if ( !fBlockInt
6781 && !fBlockSti
6782 && !fBlockMovSS)
6783 {
6784 uint8_t u8Interrupt;
6785 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6786 if (RT_SUCCESS(rc))
6787 {
6788 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6789 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6790 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6791
6792 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6793 }
6794 else
6795 {
6796 /** @todo Does this actually happen? If not turn it into an assertion. */
6797 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6798 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6799 }
6800 }
6801 else
6802 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6803 }
6804}
6805
6806
6807/**
6808 * Injects any pending events into the guest if the guest is in a state to
6809 * receive them.
6810 *
6811 * @returns VBox status code (informational status codes included).
6812 * @param pVCpu Pointer to the VMCPU.
6813 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6814 * out-of-sync. Make sure to update the required fields
6815 * before using them.
6816 */
6817static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6818{
6819 HMVMX_ASSERT_PREEMPT_SAFE();
6820 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6821
6822 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6823 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6824 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6825 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6826
6827 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6828 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6829 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6830 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6831 Assert(!TRPMHasTrap(pVCpu));
6832
6833 int rc = VINF_SUCCESS;
6834 if (pVCpu->hm.s.Event.fPending)
6835 {
6836#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6837 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6838 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6839 {
6840 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6841 AssertRCReturn(rc, rc);
6842 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6843 Assert(!fBlockInt);
6844 Assert(!fBlockSti);
6845 Assert(!fBlockMovSS);
6846 }
6847 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6848 {
6849 Assert(!fBlockSti);
6850 Assert(!fBlockMovSS);
6851 }
6852#endif
6853 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6854 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6855 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6856 AssertRCReturn(rc, rc);
6857
6858 /* Update the interruptibility-state as it could have been changed by
6859 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6860 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6861 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6862
6863#ifdef VBOX_WITH_STATISTICS
6864 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6865 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6866 else
6867 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6868#endif
6869 }
6870
6871 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6872 int rc2 = VINF_SUCCESS;
6873 if ( fBlockSti
6874 || fBlockMovSS)
6875 {
6876 if ( !pVCpu->hm.s.fSingleInstruction
6877 && !DBGFIsStepping(pVCpu))
6878 {
6879 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6880 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6881 {
6882 /*
6883 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6884 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6885 * See Intel spec. 27.3.4 "Saving Non-Register State".
6886 */
6887 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6888 AssertRCReturn(rc, rc);
6889 }
6890 }
6891 else
6892 {
6893 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6894 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6895 uIntrState = 0;
6896 }
6897 }
6898
6899 /*
6900 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6901 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6902 */
6903 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6904 AssertRC(rc2);
6905
6906 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6907 return rc;
6908}
6909
6910
6911/**
6912 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6913 *
6914 * @param pVCpu Pointer to the VMCPU.
6915 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6916 * out-of-sync. Make sure to update the required fields
6917 * before using them.
6918 */
6919DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6920{
6921 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6922 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6923}
6924
6925
6926/**
6927 * Injects a double-fault (#DF) exception into the VM.
6928 *
6929 * @returns VBox status code (informational status code included).
6930 * @param pVCpu Pointer to the VMCPU.
6931 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6932 * out-of-sync. Make sure to update the required fields
6933 * before using them.
6934 */
6935DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6936{
6937 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6938 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6939 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6940 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6941 puIntrState);
6942}
6943
6944
6945/**
6946 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6947 *
6948 * @param pVCpu Pointer to the VMCPU.
6949 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6950 * out-of-sync. Make sure to update the required fields
6951 * before using them.
6952 */
6953DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6954{
6955 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6956 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6957 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6958}
6959
6960
6961/**
6962 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6963 *
6964 * @param pVCpu Pointer to the VMCPU.
6965 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6966 * out-of-sync. Make sure to update the required fields
6967 * before using them.
6968 * @param cbInstr The value of RIP that is to be pushed on the guest
6969 * stack.
6970 */
6971DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6972{
6973 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6974 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6975 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6976}
6977
6978
6979/**
6980 * Injects a general-protection (#GP) fault into the VM.
6981 *
6982 * @returns VBox status code (informational status code included).
6983 * @param pVCpu Pointer to the VMCPU.
6984 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6985 * out-of-sync. Make sure to update the required fields
6986 * before using them.
6987 * @param u32ErrorCode The error code associated with the #GP.
6988 */
6989DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6990 uint32_t *puIntrState)
6991{
6992 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6993 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6994 if (fErrorCodeValid)
6995 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6996 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6997 puIntrState);
6998}
6999
7000
7001/**
7002 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7003 *
7004 * @param pVCpu Pointer to the VMCPU.
7005 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7006 * out-of-sync. Make sure to update the required fields
7007 * before using them.
7008 * @param uVector The software interrupt vector number.
7009 * @param cbInstr The value of RIP that is to be pushed on the guest
7010 * stack.
7011 */
7012DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7013{
7014 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7015 if ( uVector == X86_XCPT_BP
7016 || uVector == X86_XCPT_OF)
7017 {
7018 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7019 }
7020 else
7021 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7022 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7023}
7024
7025
7026/**
7027 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7028 * stack.
7029 *
7030 * @returns VBox status code (information status code included).
7031 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7032 * @param pVM Pointer to the VM.
7033 * @param pMixedCtx Pointer to the guest-CPU context.
7034 * @param uValue The value to push to the guest stack.
7035 */
7036DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7037{
7038 /*
7039 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7040 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7041 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7042 */
7043 if (pMixedCtx->sp == 1)
7044 return VINF_EM_RESET;
7045 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7046 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7047 AssertRCReturn(rc, rc);
7048 return rc;
7049}
7050
7051
7052/**
7053 * Injects an event into the guest upon VM-entry by updating the relevant fields
7054 * in the VM-entry area in the VMCS.
7055 *
7056 * @returns VBox status code (informational error codes included).
7057 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7058 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7059 *
7060 * @param pVCpu Pointer to the VMCPU.
7061 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7062 * be out-of-sync. Make sure to update the required
7063 * fields before using them.
7064 * @param u64IntInfo The VM-entry interruption-information field.
7065 * @param cbInstr The VM-entry instruction length in bytes (for
7066 * software interrupts, exceptions and privileged
7067 * software exceptions).
7068 * @param u32ErrCode The VM-entry exception error code.
7069 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7070 * @param puIntrState Pointer to the current guest interruptibility-state.
7071 * This interruptibility-state will be updated if
7072 * necessary. This cannot not be NULL.
7073 *
7074 * @remarks Requires CR0!
7075 * @remarks No-long-jump zone!!!
7076 */
7077static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7078 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7079{
7080 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7081 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7082 Assert(puIntrState);
7083 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7084
7085 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7086 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7087
7088#ifdef VBOX_STRICT
7089 /* Validate the error-code-valid bit for hardware exceptions. */
7090 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7091 {
7092 switch (uVector)
7093 {
7094 case X86_XCPT_PF:
7095 case X86_XCPT_DF:
7096 case X86_XCPT_TS:
7097 case X86_XCPT_NP:
7098 case X86_XCPT_SS:
7099 case X86_XCPT_GP:
7100 case X86_XCPT_AC:
7101 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7102 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7103 /* fallthru */
7104 default:
7105 break;
7106 }
7107 }
7108#endif
7109
7110 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7111 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7112 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7113
7114 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7115
7116 /* We require CR0 to check if the guest is in real-mode. */
7117 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7118 AssertRCReturn(rc, rc);
7119
7120 /*
7121 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7122 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7123 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7124 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7125 */
7126 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7127 {
7128 PVM pVM = pVCpu->CTX_SUFF(pVM);
7129 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7130 {
7131 Assert(PDMVmmDevHeapIsEnabled(pVM));
7132 Assert(pVM->hm.s.vmx.pRealModeTSS);
7133
7134 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7135 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7136 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7137 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7138 AssertRCReturn(rc, rc);
7139 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7140
7141 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7142 const size_t cbIdtEntry = sizeof(X86IDTR16);
7143 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7144 {
7145 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7146 if (uVector == X86_XCPT_DF)
7147 return VINF_EM_RESET;
7148 else if (uVector == X86_XCPT_GP)
7149 {
7150 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7151 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7152 }
7153
7154 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7155 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7156 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7157 }
7158
7159 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7160 uint16_t uGuestIp = pMixedCtx->ip;
7161 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7162 {
7163 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7164 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7165 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7166 }
7167 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7168 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7169
7170 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7171 X86IDTR16 IdtEntry;
7172 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7173 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7174 AssertRCReturn(rc, rc);
7175
7176 /* Construct the stack frame for the interrupt/exception handler. */
7177 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7178 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7179 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7180 AssertRCReturn(rc, rc);
7181
7182 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7183 if (rc == VINF_SUCCESS)
7184 {
7185 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7186 pMixedCtx->rip = IdtEntry.offSel;
7187 pMixedCtx->cs.Sel = IdtEntry.uSel;
7188 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7189 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7190 && uVector == X86_XCPT_PF)
7191 {
7192 pMixedCtx->cr2 = GCPtrFaultAddress;
7193 }
7194
7195 /* If any other guest-state bits are changed here, make sure to update
7196 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7197 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7198 | HM_CHANGED_GUEST_RIP
7199 | HM_CHANGED_GUEST_RFLAGS
7200 | HM_CHANGED_GUEST_RSP);
7201
7202 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7203 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7204 {
7205 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7206 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7207 Log4(("Clearing inhibition due to STI.\n"));
7208 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7209 }
7210 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7211
7212 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7213 it, if we are returning to ring-3 before executing guest code. */
7214 pVCpu->hm.s.Event.fPending = false;
7215 }
7216 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7217 return rc;
7218 }
7219 else
7220 {
7221 /*
7222 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7223 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7224 */
7225 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7226 }
7227 }
7228
7229 /* Validate. */
7230 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7231 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7232 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7233
7234 /* Inject. */
7235 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7236 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7237 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7238 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7239
7240 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7241 && uVector == X86_XCPT_PF)
7242 {
7243 pMixedCtx->cr2 = GCPtrFaultAddress;
7244 }
7245
7246 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7247 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7248
7249 AssertRCReturn(rc, rc);
7250 return rc;
7251}
7252
7253
7254/**
7255 * Clears the interrupt-window exiting control in the VMCS and if necessary
7256 * clears the current event in the VMCS as well.
7257 *
7258 * @returns VBox status code.
7259 * @param pVCpu Pointer to the VMCPU.
7260 *
7261 * @remarks Use this function only to clear events that have not yet been
7262 * delivered to the guest but are injected in the VMCS!
7263 * @remarks No-long-jump zone!!!
7264 */
7265static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7266{
7267 int rc;
7268 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7269
7270 /* Clear interrupt-window exiting control. */
7271 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7272 {
7273 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7274 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7275 AssertRC(rc);
7276 }
7277
7278 if (!pVCpu->hm.s.Event.fPending)
7279 return;
7280
7281#ifdef VBOX_STRICT
7282 uint32_t u32EntryInfo;
7283 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7284 AssertRC(rc);
7285 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7286#endif
7287
7288 /* Clear the entry-interruption field (including the valid bit). */
7289 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7290 AssertRC(rc);
7291
7292 /* Clear the pending debug exception field. */
7293 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7294 AssertRC(rc);
7295}
7296
7297
7298/**
7299 * Enters the VT-x session.
7300 *
7301 * @returns VBox status code.
7302 * @param pVM Pointer to the VM.
7303 * @param pVCpu Pointer to the VMCPU.
7304 * @param pCpu Pointer to the CPU info struct.
7305 */
7306VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7307{
7308 AssertPtr(pVM);
7309 AssertPtr(pVCpu);
7310 Assert(pVM->hm.s.vmx.fSupported);
7311 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7312 NOREF(pCpu);
7313
7314 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7315 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7316
7317#ifdef VBOX_STRICT
7318 /* Make sure we're in VMX root mode. */
7319 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7320 if (!(u32HostCR4 & X86_CR4_VMXE))
7321 {
7322 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7323 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7324 }
7325#endif
7326
7327 /*
7328 * Load the VCPU's VMCS as the current (and active) one.
7329 */
7330 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7331 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7332 if (RT_FAILURE(rc))
7333 return rc;
7334
7335 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7336 pVCpu->hm.s.fLeaveDone = false;
7337 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7338
7339 return VINF_SUCCESS;
7340}
7341
7342
7343/**
7344 * The thread-context callback (only on platforms which support it).
7345 *
7346 * @param enmEvent The thread-context event.
7347 * @param pVCpu Pointer to the VMCPU.
7348 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7349 * @thread EMT(pVCpu)
7350 */
7351VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7352{
7353 switch (enmEvent)
7354 {
7355 case RTTHREADCTXEVENT_PREEMPTING:
7356 {
7357 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7358 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7359 VMCPU_ASSERT_EMT(pVCpu);
7360
7361 PVM pVM = pVCpu->CTX_SUFF(pVM);
7362 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7363
7364 /* No longjmps (logger flushes, locks) in this fragile context. */
7365 VMMRZCallRing3Disable(pVCpu);
7366 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7367
7368 /*
7369 * Restore host-state (FPU, debug etc.)
7370 */
7371 if (!pVCpu->hm.s.fLeaveDone)
7372 {
7373 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7374 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7375 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7376 pVCpu->hm.s.fLeaveDone = true;
7377 }
7378
7379 /* Leave HM context, takes care of local init (term). */
7380 int rc = HMR0LeaveCpu(pVCpu);
7381 AssertRC(rc); NOREF(rc);
7382
7383 /* Restore longjmp state. */
7384 VMMRZCallRing3Enable(pVCpu);
7385 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7386 break;
7387 }
7388
7389 case RTTHREADCTXEVENT_RESUMED:
7390 {
7391 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7392 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7393 VMCPU_ASSERT_EMT(pVCpu);
7394
7395 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7396 VMMRZCallRing3Disable(pVCpu);
7397 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7398
7399 /* Initialize the bare minimum state required for HM. This takes care of
7400 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7401 int rc = HMR0EnterCpu(pVCpu);
7402 AssertRC(rc);
7403 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7404
7405 /* Load the active VMCS as the current one. */
7406 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7407 {
7408 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7409 AssertRC(rc); NOREF(rc);
7410 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7411 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7412 }
7413 pVCpu->hm.s.fLeaveDone = false;
7414
7415 /* Restore longjmp state. */
7416 VMMRZCallRing3Enable(pVCpu);
7417 break;
7418 }
7419
7420 default:
7421 break;
7422 }
7423}
7424
7425
7426/**
7427 * Saves the host state in the VMCS host-state.
7428 * Sets up the VM-exit MSR-load area.
7429 *
7430 * The CPU state will be loaded from these fields on every successful VM-exit.
7431 *
7432 * @returns VBox status code.
7433 * @param pVM Pointer to the VM.
7434 * @param pVCpu Pointer to the VMCPU.
7435 *
7436 * @remarks No-long-jump zone!!!
7437 */
7438static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7439{
7440 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7441
7442 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7443 return VINF_SUCCESS;
7444
7445 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7446 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7447
7448 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7449 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7450
7451 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7452 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7453
7454 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7455 return rc;
7456}
7457
7458
7459/**
7460 * Saves the host state in the VMCS host-state.
7461 *
7462 * @returns VBox status code.
7463 * @param pVM Pointer to the VM.
7464 * @param pVCpu Pointer to the VMCPU.
7465 *
7466 * @remarks No-long-jump zone!!!
7467 */
7468VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7469{
7470 AssertPtr(pVM);
7471 AssertPtr(pVCpu);
7472
7473 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7474
7475 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7476 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7477 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7478 return hmR0VmxSaveHostState(pVM, pVCpu);
7479}
7480
7481
7482/**
7483 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7484 * loaded from these fields on every successful VM-entry.
7485 *
7486 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7487 * Sets up the VM-entry controls.
7488 * Sets up the appropriate VMX non-root function to execute guest code based on
7489 * the guest CPU mode.
7490 *
7491 * @returns VBox status code.
7492 * @param pVM Pointer to the VM.
7493 * @param pVCpu Pointer to the VMCPU.
7494 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7495 * out-of-sync. Make sure to update the required fields
7496 * before using them.
7497 *
7498 * @remarks No-long-jump zone!!!
7499 */
7500static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7501{
7502 AssertPtr(pVM);
7503 AssertPtr(pVCpu);
7504 AssertPtr(pMixedCtx);
7505 HMVMX_ASSERT_PREEMPT_SAFE();
7506
7507#ifdef LOG_ENABLED
7508 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7509 * probably not initialized yet? Anyway this will do for now.
7510 *
7511 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7512 * interface and disable ring-3 calls when thread-context hooks are not
7513 * available. */
7514 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7515 VMMR0LogFlushDisable(pVCpu);
7516#endif
7517
7518 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7519
7520 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7521
7522 /* Determine real-on-v86 mode. */
7523 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7524 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7525 && CPUMIsGuestInRealModeEx(pMixedCtx))
7526 {
7527 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7528 }
7529
7530 /*
7531 * Load the guest-state into the VMCS.
7532 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7533 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7534 */
7535 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7536 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7537
7538 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7539 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7540 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7541
7542 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7543 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7544 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7545
7546 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7547 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7548
7549 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7550 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7551
7552 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7553 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7554 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7555
7556 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7557 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7558
7559 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7560 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7561
7562 /*
7563 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7564 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7565 */
7566 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7567 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7568
7569 /* Clear any unused and reserved bits. */
7570 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7571
7572#ifdef LOG_ENABLED
7573 /* Only reenable log-flushing if the caller has it enabled. */
7574 if (!fCallerDisabledLogFlush)
7575 VMMR0LogFlushEnable(pVCpu);
7576#endif
7577
7578 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7579 return rc;
7580}
7581
7582
7583/**
7584 * Loads the state shared between the host and guest into the VMCS.
7585 *
7586 * @param pVM Pointer to the VM.
7587 * @param pVCpu Pointer to the VMCPU.
7588 * @param pCtx Pointer to the guest-CPU context.
7589 *
7590 * @remarks No-long-jump zone!!!
7591 */
7592static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7593{
7594 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7595 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7596
7597 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7598 {
7599 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7600 AssertRC(rc);
7601 }
7602
7603 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7604 {
7605 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7606 AssertRC(rc);
7607
7608 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7609 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7610 {
7611 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7612 AssertRC(rc);
7613 }
7614 }
7615
7616 AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7617 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7618}
7619
7620
7621/**
7622 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7623 *
7624 * @param pVM Pointer to the VM.
7625 * @param pVCpu Pointer to the VMCPU.
7626 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7627 * out-of-sync. Make sure to update the required fields
7628 * before using them.
7629 */
7630DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7631{
7632 HMVMX_ASSERT_PREEMPT_SAFE();
7633
7634 Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7635#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7636 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7637#endif
7638
7639 if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7640 {
7641 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7642 AssertRC(rc);
7643 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7644 }
7645 else if (VMCPU_HMCF_VALUE(pVCpu))
7646 {
7647 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7648 AssertRC(rc);
7649 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7650 }
7651
7652 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7653 AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7654 || VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
7655 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7656
7657#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7658 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7659 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7660 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7661#endif
7662}
7663
7664
7665/**
7666 * Does the preparations before executing guest code in VT-x.
7667 *
7668 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7669 * recompiler. We must be cautious what we do here regarding committing
7670 * guest-state information into the VMCS assuming we assuredly execute the
7671 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7672 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7673 * so that the recompiler can (and should) use them when it resumes guest
7674 * execution. Otherwise such operations must be done when we can no longer
7675 * exit to ring-3.
7676 *
7677 * @returns Strict VBox status code.
7678 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7679 * have been disabled.
7680 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7681 * double-fault into the guest.
7682 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7683 *
7684 * @param pVM Pointer to the VM.
7685 * @param pVCpu Pointer to the VMCPU.
7686 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7687 * out-of-sync. Make sure to update the required fields
7688 * before using them.
7689 * @param pVmxTransient Pointer to the VMX transient structure.
7690 *
7691 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7692 * interrupts will be disabled.
7693 */
7694static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7695{
7696 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7697
7698#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7699 PGMRZDynMapFlushAutoSet(pVCpu);
7700#endif
7701
7702 /* Check force flag actions that might require us to go back to ring-3. */
7703 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7704 if (rc != VINF_SUCCESS)
7705 return rc;
7706
7707#ifndef IEM_VERIFICATION_MODE_FULL
7708 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7709 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7710 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7711 {
7712 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7713 RTGCPHYS GCPhysApicBase;
7714 GCPhysApicBase = pMixedCtx->msrApicBase;
7715 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7716
7717 /* Unalias any existing mapping. */
7718 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7719 AssertRCReturn(rc, rc);
7720
7721 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7722 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7723 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7724 AssertRCReturn(rc, rc);
7725
7726 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7727 }
7728#endif /* !IEM_VERIFICATION_MODE_FULL */
7729
7730 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7731 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7732
7733 /*
7734 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7735 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7736 */
7737 if (TRPMHasTrap(pVCpu))
7738 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7739 else if (!pVCpu->hm.s.Event.fPending)
7740 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7741
7742 /*
7743 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7744 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7745 */
7746 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7747 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7748 {
7749 Assert(rc == VINF_EM_RESET);
7750 return rc;
7751 }
7752
7753 /*
7754 * No longjmps to ring-3 from this point on!!!
7755 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7756 * This also disables flushing of the R0-logger instance (if any).
7757 */
7758 VMMRZCallRing3Disable(pVCpu);
7759
7760 /*
7761 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7762 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7763 *
7764 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7765 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7766 *
7767 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7768 * executing guest code.
7769 */
7770 pVmxTransient->uEflags = ASMIntDisableFlags();
7771 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7772 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7773 {
7774 hmR0VmxClearEventVmcs(pVCpu);
7775 ASMSetFlags(pVmxTransient->uEflags);
7776 VMMRZCallRing3Enable(pVCpu);
7777 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7778 return VINF_EM_RAW_TO_R3;
7779 }
7780 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7781 {
7782 hmR0VmxClearEventVmcs(pVCpu);
7783 ASMSetFlags(pVmxTransient->uEflags);
7784 VMMRZCallRing3Enable(pVCpu);
7785 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7786 return VINF_EM_RAW_INTERRUPT;
7787 }
7788
7789 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7790 pVCpu->hm.s.Event.fPending = false;
7791
7792 return VINF_SUCCESS;
7793}
7794
7795
7796/**
7797 * Prepares to run guest code in VT-x and we've committed to doing so. This
7798 * means there is no backing out to ring-3 or anywhere else at this
7799 * point.
7800 *
7801 * @param pVM Pointer to the VM.
7802 * @param pVCpu Pointer to the VMCPU.
7803 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7804 * out-of-sync. Make sure to update the required fields
7805 * before using them.
7806 * @param pVmxTransient Pointer to the VMX transient structure.
7807 *
7808 * @remarks Called with preemption disabled.
7809 * @remarks No-long-jump zone!!!
7810 */
7811static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7812{
7813 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7814 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7815 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7816
7817 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7818 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7819
7820 /*
7821 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7822 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7823 * Reload only the necessary state, the assertion will catch if other parts of the code
7824 * change.
7825 */
7826 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7827 {
7828 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7829 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7830 }
7831
7832#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7833 if (!CPUMIsGuestFPUStateActive(pVCpu))
7834 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7835 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7836#endif
7837
7838 if ( pVCpu->hm.s.fUseGuestFpu
7839 && !CPUMIsGuestFPUStateActive(pVCpu))
7840 {
7841 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7842 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
7843 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7844 }
7845
7846 /*
7847 * Load the host state bits as we may've been preempted (only happens when
7848 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7849 */
7850 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7851 {
7852 /* This ASSUMES that pfnStartVM has been set up already. */
7853 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7854 AssertRC(rc);
7855 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7856 }
7857 Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7858
7859 /*
7860 * Load the state shared between host and guest (FPU, debug).
7861 */
7862 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7863 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7864 AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7865
7866 /* Store status of the shared guest-host state at the time of VM-entry. */
7867#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7868 if (CPUMIsGuestInLongModeEx(pMixedCtx))
7869 {
7870 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7871 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7872 }
7873 else
7874#endif
7875 {
7876 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7877 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7878 }
7879 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7880
7881 /*
7882 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7883 */
7884 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7885 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7886
7887 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7888 RTCPUID idCurrentCpu = pCpu->idCpu;
7889 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7890 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7891 {
7892 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7893 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7894 }
7895
7896 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7897 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7898 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7899 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7900
7901 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7902
7903 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7904 to start executing. */
7905
7906#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7907 /*
7908 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7909 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7910 */
7911 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7912 {
7913 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7914 uint64_t u64HostTscAux = 0;
7915 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7916 AssertRC(rc2);
7917 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7918 }
7919#endif
7920}
7921
7922
7923/**
7924 * Performs some essential restoration of state after running guest code in
7925 * VT-x.
7926 *
7927 * @param pVM Pointer to the VM.
7928 * @param pVCpu Pointer to the VMCPU.
7929 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7930 * out-of-sync. Make sure to update the required fields
7931 * before using them.
7932 * @param pVmxTransient Pointer to the VMX transient structure.
7933 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7934 *
7935 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7936 *
7937 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7938 * unconditionally when it is safe to do so.
7939 */
7940static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7941{
7942 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7943
7944 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7945 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7946 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7947 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7948 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7949
7950 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7951 {
7952#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7953 /* Restore host's TSC_AUX. */
7954 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7955 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7956#endif
7957 /** @todo Find a way to fix hardcoding a guestimate. */
7958 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7959 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7960 }
7961
7962 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7963 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7964 Assert(!(ASMGetFlags() & X86_EFL_IF));
7965 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7966
7967#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7968 if (CPUMIsGuestFPUStateActive(pVCpu))
7969 {
7970 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7971 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7972 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7973 }
7974#endif
7975
7976 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7977 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7978 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7979 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7980
7981 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7982 uint32_t uExitReason;
7983 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7984 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7985 AssertRC(rc);
7986 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7987 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
7988
7989 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7990 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7991 {
7992 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7993 pVmxTransient->fVMEntryFailed));
7994 return;
7995 }
7996
7997 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7998 {
7999 /* Update the guest interruptibility-state from the VMCS. */
8000 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8001#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8002 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8003 AssertRC(rc);
8004#endif
8005 /*
8006 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8007 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8008 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8009 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8010 */
8011 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8012 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8013 {
8014 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8015 AssertRC(rc);
8016 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8017 }
8018 }
8019}
8020
8021
8022
8023/**
8024 * Runs the guest code using VT-x the normal way.
8025 *
8026 * @returns VBox status code.
8027 * @param pVM Pointer to the VM.
8028 * @param pVCpu Pointer to the VMCPU.
8029 * @param pCtx Pointer to the guest-CPU context.
8030 *
8031 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
8032 * @remarks Called with preemption disabled.
8033 */
8034static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8035{
8036 VMXTRANSIENT VmxTransient;
8037 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8038 int rc = VERR_INTERNAL_ERROR_5;
8039 uint32_t cLoops = 0;
8040
8041 for (;; cLoops++)
8042 {
8043 Assert(!HMR0SuspendPending());
8044 HMVMX_ASSERT_CPU_SAFE();
8045
8046 /* Preparatory work for running guest code, this may force us to return
8047 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8048 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8049 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8050 if (rc != VINF_SUCCESS)
8051 break;
8052
8053 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8054 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8055 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8056
8057 /* Restore any residual host-state and save any bits shared between host
8058 and guest into the guest-CPU state. Re-enables interrupts! */
8059 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8060
8061 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8062 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8063 {
8064 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8065 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8066 return rc;
8067 }
8068
8069 /* Handle the VM-exit. */
8070 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8072 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8073 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8074 HMVMX_START_EXIT_DISPATCH_PROF();
8075#ifdef HMVMX_USE_FUNCTION_TABLE
8076 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8077#else
8078 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8079#endif
8080 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8081 if (rc != VINF_SUCCESS)
8082 break;
8083 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8084 {
8085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8086 rc = VINF_EM_RAW_INTERRUPT;
8087 break;
8088 }
8089 }
8090
8091 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8092 return rc;
8093}
8094
8095
8096/**
8097 * Single steps guest code using VT-x.
8098 *
8099 * @returns VBox status code.
8100 * @param pVM Pointer to the VM.
8101 * @param pVCpu Pointer to the VMCPU.
8102 * @param pCtx Pointer to the guest-CPU context.
8103 *
8104 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8105 * @remarks Called with preemption disabled.
8106 */
8107static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8108{
8109 VMXTRANSIENT VmxTransient;
8110 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8111 int rc = VERR_INTERNAL_ERROR_5;
8112 uint32_t cLoops = 0;
8113 uint16_t uCsStart = pCtx->cs.Sel;
8114 uint64_t uRipStart = pCtx->rip;
8115
8116 for (;; cLoops++)
8117 {
8118 Assert(!HMR0SuspendPending());
8119 HMVMX_ASSERT_CPU_SAFE();
8120
8121 /* Preparatory work for running guest code, this may force us to return
8122 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8123 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8124 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8125 if (rc != VINF_SUCCESS)
8126 break;
8127
8128 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8129 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8130 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8131
8132 /* Restore any residual host-state and save any bits shared between host
8133 and guest into the guest-CPU state. Re-enables interrupts! */
8134 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8135
8136 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8137 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8138 {
8139 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8140 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8141 return rc;
8142 }
8143
8144 /* Handle the VM-exit. */
8145 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8146 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8147 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8148 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8149 HMVMX_START_EXIT_DISPATCH_PROF();
8150#ifdef HMVMX_USE_FUNCTION_TABLE
8151 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8152#else
8153 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8154#endif
8155 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8156 if (rc != VINF_SUCCESS)
8157 break;
8158 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8159 {
8160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8161 rc = VINF_EM_RAW_INTERRUPT;
8162 break;
8163 }
8164
8165 /*
8166 * Did the RIP change, if so, consider it a single step.
8167 * Otherwise, make sure one of the TFs gets set.
8168 */
8169 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8170 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8171 AssertRCReturn(rc2, rc2);
8172 if ( pCtx->rip != uRipStart
8173 || pCtx->cs.Sel != uCsStart)
8174 {
8175 rc = VINF_EM_DBG_STEPPED;
8176 break;
8177 }
8178 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8179 }
8180
8181 /*
8182 * Clear the X86_EFL_TF if necessary.
8183 */
8184 if (pVCpu->hm.s.fClearTrapFlag)
8185 {
8186 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8187 AssertRCReturn(rc2, rc2);
8188 pVCpu->hm.s.fClearTrapFlag = false;
8189 pCtx->eflags.Bits.u1TF = 0;
8190 }
8191 /** @todo there seems to be issues with the resume flag when the monitor trap
8192 * flag is pending without being used. Seen early in bios init when
8193 * accessing APIC page in prot mode. */
8194
8195 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8196 return rc;
8197}
8198
8199
8200/**
8201 * Runs the guest code using VT-x.
8202 *
8203 * @returns VBox status code.
8204 * @param pVM Pointer to the VM.
8205 * @param pVCpu Pointer to the VMCPU.
8206 * @param pCtx Pointer to the guest-CPU context.
8207 *
8208 * @remarks Called with preemption disabled.
8209 */
8210VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8211{
8212 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8213 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8214 HMVMX_ASSERT_PREEMPT_SAFE();
8215
8216 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8217
8218 int rc;
8219 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8220 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8221 else
8222 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8223
8224 if (rc == VERR_EM_INTERPRETER)
8225 rc = VINF_EM_RAW_EMULATE_INSTR;
8226 else if (rc == VINF_EM_RESET)
8227 rc = VINF_EM_TRIPLE_FAULT;
8228
8229 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8230 if (RT_FAILURE(rc2))
8231 {
8232 pVCpu->hm.s.u32HMError = rc;
8233 rc = rc2;
8234 }
8235 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8236 return rc;
8237}
8238
8239
8240#ifndef HMVMX_USE_FUNCTION_TABLE
8241DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8242{
8243 int rc;
8244 switch (rcReason)
8245 {
8246 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8247 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8248 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8249 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8250 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8251 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8252 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8253 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8254 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8255 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8256 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8257 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8258 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8259 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8260 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8261 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8262 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8263 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8264 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8265 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8266 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8267 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8268 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8269 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8270 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8271 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8272 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8273 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8274 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8275 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8276 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8277 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8278 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8279
8280 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8281 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8282 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8283 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8284 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8285 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8286 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8287 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8288 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8289
8290 case VMX_EXIT_VMCALL:
8291 case VMX_EXIT_VMCLEAR:
8292 case VMX_EXIT_VMLAUNCH:
8293 case VMX_EXIT_VMPTRLD:
8294 case VMX_EXIT_VMPTRST:
8295 case VMX_EXIT_VMREAD:
8296 case VMX_EXIT_VMRESUME:
8297 case VMX_EXIT_VMWRITE:
8298 case VMX_EXIT_VMXOFF:
8299 case VMX_EXIT_VMXON:
8300 case VMX_EXIT_INVEPT:
8301 case VMX_EXIT_INVVPID:
8302 case VMX_EXIT_VMFUNC:
8303 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8304 break;
8305 default:
8306 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8307 break;
8308 }
8309 return rc;
8310}
8311#endif
8312
8313#ifdef DEBUG
8314/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8315# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8316 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8317
8318# define HMVMX_ASSERT_PREEMPT_CPUID() \
8319 do \
8320 { \
8321 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8322 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8323 } while (0)
8324
8325# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8326 do { \
8327 AssertPtr(pVCpu); \
8328 AssertPtr(pMixedCtx); \
8329 AssertPtr(pVmxTransient); \
8330 Assert(pVmxTransient->fVMEntryFailed == false); \
8331 Assert(ASMIntAreEnabled()); \
8332 HMVMX_ASSERT_PREEMPT_SAFE(); \
8333 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8334 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8335 HMVMX_ASSERT_PREEMPT_SAFE(); \
8336 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8337 HMVMX_ASSERT_PREEMPT_CPUID(); \
8338 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8339 } while (0)
8340
8341# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8342 do { \
8343 Log4Func(("\n")); \
8344 } while(0)
8345#else /* Release builds */
8346# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8347# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8348#endif
8349
8350
8351/**
8352 * Advances the guest RIP after reading it from the VMCS.
8353 *
8354 * @returns VBox status code.
8355 * @param pVCpu Pointer to the VMCPU.
8356 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8357 * out-of-sync. Make sure to update the required fields
8358 * before using them.
8359 * @param pVmxTransient Pointer to the VMX transient structure.
8360 *
8361 * @remarks No-long-jump zone!!!
8362 */
8363DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8364{
8365 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8366 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8367 AssertRCReturn(rc, rc);
8368
8369 pMixedCtx->rip += pVmxTransient->cbInstr;
8370 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8371 return rc;
8372}
8373
8374
8375/**
8376 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8377 * and update error record fields accordingly.
8378 *
8379 * @return VMX_IGS_* return codes.
8380 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8381 * wrong with the guest state.
8382 *
8383 * @param pVM Pointer to the VM.
8384 * @param pVCpu Pointer to the VMCPU.
8385 * @param pCtx Pointer to the guest-CPU state.
8386 */
8387static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8388{
8389#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8390#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8391 uError = (err); \
8392 break; \
8393 } else do {} while (0)
8394/* Duplicate of IEM_IS_CANONICAL(). */
8395#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8396
8397 int rc;
8398 uint32_t uError = VMX_IGS_ERROR;
8399 uint32_t u32Val;
8400 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8401
8402 do
8403 {
8404 /*
8405 * CR0.
8406 */
8407 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8408 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8409 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8410 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8411 if (fUnrestrictedGuest)
8412 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8413
8414 uint32_t u32GuestCR0;
8415 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8416 AssertRCBreak(rc);
8417 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8418 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8419 if ( !fUnrestrictedGuest
8420 && (u32GuestCR0 & X86_CR0_PG)
8421 && !(u32GuestCR0 & X86_CR0_PE))
8422 {
8423 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8424 }
8425
8426 /*
8427 * CR4.
8428 */
8429 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8430 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8431
8432 uint32_t u32GuestCR4;
8433 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8434 AssertRCBreak(rc);
8435 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8436 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8437
8438 /*
8439 * IA32_DEBUGCTL MSR.
8440 */
8441 uint64_t u64Val;
8442 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8443 AssertRCBreak(rc);
8444 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8445 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8446 {
8447 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8448 }
8449 uint64_t u64DebugCtlMsr = u64Val;
8450
8451#ifdef VBOX_STRICT
8452 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8453 AssertRCBreak(rc);
8454 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8455#endif
8456 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8457
8458 /*
8459 * RIP and RFLAGS.
8460 */
8461 uint32_t u32Eflags;
8462#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8463 if (HMVMX_IS_64BIT_HOST_MODE())
8464 {
8465 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8466 AssertRCBreak(rc);
8467 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8468 if ( !fLongModeGuest
8469 || !pCtx->cs.Attr.n.u1Long)
8470 {
8471 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8472 }
8473 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8474 * must be identical if the "IA32e mode guest" VM-entry control is 1
8475 * and CS.L is 1. No check applies if the CPU supports 64
8476 * linear-address bits. */
8477
8478 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8479 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8480 AssertRCBreak(rc);
8481 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8482 VMX_IGS_RFLAGS_RESERVED);
8483 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8484 u32Eflags = u64Val;
8485 }
8486 else
8487#endif
8488 {
8489 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8490 AssertRCBreak(rc);
8491 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8492 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8493 }
8494
8495 if ( fLongModeGuest
8496 || ( fUnrestrictedGuest
8497 && !(u32GuestCR0 & X86_CR0_PE)))
8498 {
8499 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8500 }
8501
8502 uint32_t u32EntryInfo;
8503 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8504 AssertRCBreak(rc);
8505 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8506 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8507 {
8508 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8509 }
8510
8511 /*
8512 * 64-bit checks.
8513 */
8514#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8515 if (HMVMX_IS_64BIT_HOST_MODE())
8516 {
8517 if ( fLongModeGuest
8518 && !fUnrestrictedGuest)
8519 {
8520 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8521 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8522 }
8523
8524 if ( !fLongModeGuest
8525 && (u32GuestCR4 & X86_CR4_PCIDE))
8526 {
8527 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8528 }
8529
8530 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8531 * 51:32 beyond the processor's physical-address width are 0. */
8532
8533 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8534 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8535 {
8536 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8537 }
8538
8539 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8540 AssertRCBreak(rc);
8541 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8542
8543 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8544 AssertRCBreak(rc);
8545 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8546 }
8547#endif
8548
8549 /*
8550 * PERF_GLOBAL MSR.
8551 */
8552 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8553 {
8554 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8555 AssertRCBreak(rc);
8556 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8557 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8558 }
8559
8560 /*
8561 * PAT MSR.
8562 */
8563 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8564 {
8565 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8566 AssertRCBreak(rc);
8567 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8568 for (unsigned i = 0; i < 8; i++)
8569 {
8570 uint8_t u8Val = (u64Val & 0x7);
8571 if ( u8Val != 0 /* UC */
8572 || u8Val != 1 /* WC */
8573 || u8Val != 4 /* WT */
8574 || u8Val != 5 /* WP */
8575 || u8Val != 6 /* WB */
8576 || u8Val != 7 /* UC- */)
8577 {
8578 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8579 }
8580 u64Val >>= 3;
8581 }
8582 }
8583
8584 /*
8585 * EFER MSR.
8586 */
8587 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8588 {
8589 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8590 AssertRCBreak(rc);
8591 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8592 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8593 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8594 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8595 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8596 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8597 }
8598
8599 /*
8600 * Segment registers.
8601 */
8602 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8603 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8604 if (!(u32Eflags & X86_EFL_VM))
8605 {
8606 /* CS */
8607 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8608 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8609 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8610 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8611 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8612 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8613 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8614 /* CS cannot be loaded with NULL in protected mode. */
8615 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8616 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8617 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8618 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8619 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8620 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8621 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8622 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8623 else
8624 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8625
8626 /* SS */
8627 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8628 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8629 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8630 if ( !(pCtx->cr0 & X86_CR0_PE)
8631 || pCtx->cs.Attr.n.u4Type == 3)
8632 {
8633 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8634 }
8635 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8636 {
8637 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8638 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8639 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8640 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8641 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8642 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8643 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8644 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8645 }
8646
8647 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8648 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8649 {
8650 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8651 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8652 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8653 || pCtx->ds.Attr.n.u4Type > 11
8654 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8655 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8656 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8657 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8658 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8659 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8660 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8661 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8662 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8663 }
8664 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8665 {
8666 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8667 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8668 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8669 || pCtx->es.Attr.n.u4Type > 11
8670 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8671 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8672 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8673 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8674 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8675 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8676 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8677 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8678 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8679 }
8680 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8681 {
8682 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8683 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8684 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8685 || pCtx->fs.Attr.n.u4Type > 11
8686 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8687 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8688 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8689 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8690 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8691 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8692 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8693 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8694 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8695 }
8696 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8697 {
8698 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8699 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8700 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8701 || pCtx->gs.Attr.n.u4Type > 11
8702 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8703 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8704 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8705 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8706 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8707 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8708 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8709 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8710 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8711 }
8712 /* 64-bit capable CPUs. */
8713#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8714 if (HMVMX_IS_64BIT_HOST_MODE())
8715 {
8716 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8717 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8718 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8719 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8720 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8721 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8722 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8723 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8724 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8725 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8726 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8727 }
8728#endif
8729 }
8730 else
8731 {
8732 /* V86 mode checks. */
8733 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8734 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8735 {
8736 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8737 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8738 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8739 }
8740 else
8741 {
8742 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8743 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8744 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8745 }
8746
8747 /* CS */
8748 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8749 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8750 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8751 /* SS */
8752 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8753 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8754 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8755 /* DS */
8756 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8757 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8758 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8759 /* ES */
8760 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8761 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8762 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8763 /* FS */
8764 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8765 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8766 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8767 /* GS */
8768 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8769 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8770 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8771 /* 64-bit capable CPUs. */
8772#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8773 if (HMVMX_IS_64BIT_HOST_MODE())
8774 {
8775 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8776 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8777 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8778 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8779 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8780 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8781 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8782 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8783 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8784 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8785 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8786 }
8787#endif
8788 }
8789
8790 /*
8791 * TR.
8792 */
8793 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8794 /* 64-bit capable CPUs. */
8795#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8796 if (HMVMX_IS_64BIT_HOST_MODE())
8797 {
8798 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8799 }
8800#endif
8801 if (fLongModeGuest)
8802 {
8803 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8804 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8805 }
8806 else
8807 {
8808 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8809 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8810 VMX_IGS_TR_ATTR_TYPE_INVALID);
8811 }
8812 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8813 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8814 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8815 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8816 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8817 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8818 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8819 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8820
8821 /*
8822 * GDTR and IDTR.
8823 */
8824#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8825 if (HMVMX_IS_64BIT_HOST_MODE())
8826 {
8827 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8828 AssertRCBreak(rc);
8829 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8830
8831 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8832 AssertRCBreak(rc);
8833 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8834 }
8835#endif
8836
8837 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8838 AssertRCBreak(rc);
8839 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8840
8841 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8842 AssertRCBreak(rc);
8843 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8844
8845 /*
8846 * Guest Non-Register State.
8847 */
8848 /* Activity State. */
8849 uint32_t u32ActivityState;
8850 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8851 AssertRCBreak(rc);
8852 HMVMX_CHECK_BREAK( !u32ActivityState
8853 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8854 VMX_IGS_ACTIVITY_STATE_INVALID);
8855 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8856 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8857 uint32_t u32IntrState;
8858 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8859 AssertRCBreak(rc);
8860 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8861 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8862 {
8863 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8864 }
8865
8866 /** @todo Activity state and injecting interrupts. Left as a todo since we
8867 * currently don't use activity states but ACTIVE. */
8868
8869 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8870 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8871
8872 /* Guest interruptibility-state. */
8873 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8874 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8875 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8876 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8877 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8878 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8879 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8880 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8881 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8882 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
8883 {
8884 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8885 {
8886 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8887 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8888 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8889 }
8890 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8891 {
8892 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8893 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8894 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8895 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8896 }
8897 }
8898 /** @todo Assumes the processor is not in SMM. */
8899 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8900 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8901 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8902 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8903 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8904 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8905 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8906 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8907 {
8908 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8909 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8910 }
8911
8912 /* Pending debug exceptions. */
8913 if (HMVMX_IS_64BIT_HOST_MODE())
8914 {
8915 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8916 AssertRCBreak(rc);
8917 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8918 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8919 u32Val = u64Val; /* For pending debug exceptions checks below. */
8920 }
8921 else
8922 {
8923 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8924 AssertRCBreak(rc);
8925 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8926 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8927 }
8928
8929 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8930 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8931 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8932 {
8933 if ( (u32Eflags & X86_EFL_TF)
8934 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8935 {
8936 /* Bit 14 is PendingDebug.BS. */
8937 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8938 }
8939 if ( !(u32Eflags & X86_EFL_TF)
8940 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8941 {
8942 /* Bit 14 is PendingDebug.BS. */
8943 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8944 }
8945 }
8946
8947 /* VMCS link pointer. */
8948 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8949 AssertRCBreak(rc);
8950 if (u64Val != UINT64_C(0xffffffffffffffff))
8951 {
8952 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8953 /** @todo Bits beyond the processor's physical-address width MBZ. */
8954 /** @todo 32-bit located in memory referenced by value of this field (as a
8955 * physical address) must contain the processor's VMCS revision ID. */
8956 /** @todo SMM checks. */
8957 }
8958
8959 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8960
8961 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8962 if (uError == VMX_IGS_ERROR)
8963 uError = VMX_IGS_REASON_NOT_FOUND;
8964 } while (0);
8965
8966 pVCpu->hm.s.u32HMError = uError;
8967 return uError;
8968
8969#undef HMVMX_ERROR_BREAK
8970#undef HMVMX_CHECK_BREAK
8971#undef HMVMX_IS_CANONICAL
8972}
8973
8974/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8975/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8976/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8977
8978/** @name VM-exit handlers.
8979 * @{
8980 */
8981
8982/**
8983 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8984 */
8985HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8986{
8987 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8989 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8990#if HC_ARCH_BITS == 64
8991 Assert(ASMIntAreEnabled());
8992 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8993 return VINF_SUCCESS;
8994#endif
8995 return VINF_EM_RAW_INTERRUPT;
8996}
8997
8998
8999/**
9000 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9001 */
9002HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9003{
9004 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9005 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9006
9007 int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
9008 AssertRCReturn(rc, rc);
9009
9010 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9011 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9012 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9013 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9014
9015 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9016 {
9017 /*
9018 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9019 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9020 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9021 *
9022 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9023 */
9024 VMXDispatchHostNmi();
9025 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9026 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9027 return VINF_SUCCESS;
9028 }
9029
9030 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9031 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9032 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9033 {
9034 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9035 return VINF_SUCCESS;
9036 }
9037 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9038 {
9039 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9040 return rc;
9041 }
9042
9043 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9044 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9045 switch (uIntType)
9046 {
9047 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9048 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
9049 /* no break */
9050 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9051 {
9052 switch (uVector)
9053 {
9054 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9055 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9056 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9057 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9058 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9059 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9060#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9061 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9062 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9063 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9064 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9065 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9066 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9067 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9068 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9069 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9070 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9071#endif
9072 default:
9073 {
9074 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9075 AssertRCReturn(rc, rc);
9076
9077 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9078 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9079 {
9080 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9081 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9082 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9083
9084 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9085 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
9086 AssertRCReturn(rc, rc);
9087 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9088 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9089 0 /* GCPtrFaultAddress */);
9090 AssertRCReturn(rc, rc);
9091 }
9092 else
9093 {
9094 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9095 pVCpu->hm.s.u32HMError = uVector;
9096 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9097 }
9098 break;
9099 }
9100 }
9101 break;
9102 }
9103
9104 default:
9105 {
9106 pVCpu->hm.s.u32HMError = uExitIntInfo;
9107 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9108 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9109 break;
9110 }
9111 }
9112 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9113 return rc;
9114}
9115
9116
9117/**
9118 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9119 */
9120HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9121{
9122 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9123
9124 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9125 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9126 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9127 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9128 AssertRCReturn(rc, rc);
9129
9130 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9132 return VINF_SUCCESS;
9133}
9134
9135
9136/**
9137 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9138 */
9139HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9140{
9141 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9142 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9143 HMVMX_RETURN_UNEXPECTED_EXIT();
9144}
9145
9146
9147/**
9148 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9149 */
9150HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9151{
9152 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9153 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9154 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9155}
9156
9157
9158/**
9159 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9160 */
9161HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9162{
9163 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9165 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9166}
9167
9168
9169/**
9170 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9171 */
9172HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9173{
9174 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9175 PVM pVM = pVCpu->CTX_SUFF(pVM);
9176 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9177 if (RT_LIKELY(rc == VINF_SUCCESS))
9178 {
9179 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9180 Assert(pVmxTransient->cbInstr == 2);
9181 }
9182 else
9183 {
9184 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9185 rc = VERR_EM_INTERPRETER;
9186 }
9187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9188 return rc;
9189}
9190
9191
9192/**
9193 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9194 */
9195HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9196{
9197 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9198 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9199 AssertRCReturn(rc, rc);
9200
9201 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9202 return VINF_EM_RAW_EMULATE_INSTR;
9203
9204 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9205 HMVMX_RETURN_UNEXPECTED_EXIT();
9206}
9207
9208
9209/**
9210 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9211 */
9212HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9213{
9214 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9215 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9216 AssertRCReturn(rc, rc);
9217
9218 PVM pVM = pVCpu->CTX_SUFF(pVM);
9219 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9220 if (RT_LIKELY(rc == VINF_SUCCESS))
9221 {
9222 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9223 Assert(pVmxTransient->cbInstr == 2);
9224 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9225 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9226 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9227 }
9228 else
9229 {
9230 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9231 rc = VERR_EM_INTERPRETER;
9232 }
9233 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9234 return rc;
9235}
9236
9237
9238/**
9239 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9240 */
9241HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9242{
9243 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9244 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9245 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9246 AssertRCReturn(rc, rc);
9247
9248 PVM pVM = pVCpu->CTX_SUFF(pVM);
9249 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9250 if (RT_LIKELY(rc == VINF_SUCCESS))
9251 {
9252 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9253 Assert(pVmxTransient->cbInstr == 3);
9254 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9255 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9256 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9257 }
9258 else
9259 {
9260 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9261 rc = VERR_EM_INTERPRETER;
9262 }
9263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9264 return rc;
9265}
9266
9267
9268/**
9269 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9270 */
9271HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9272{
9273 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9274 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9275 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9276 AssertRCReturn(rc, rc);
9277
9278 PVM pVM = pVCpu->CTX_SUFF(pVM);
9279 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9280 if (RT_LIKELY(rc == VINF_SUCCESS))
9281 {
9282 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9283 Assert(pVmxTransient->cbInstr == 2);
9284 }
9285 else
9286 {
9287 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9288 rc = VERR_EM_INTERPRETER;
9289 }
9290 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9291 return rc;
9292}
9293
9294
9295/**
9296 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9297 */
9298HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9299{
9300 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9301 PVM pVM = pVCpu->CTX_SUFF(pVM);
9302 Assert(!pVM->hm.s.fNestedPaging);
9303
9304 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9305 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9306 AssertRCReturn(rc, rc);
9307
9308 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9309 rc = VBOXSTRICTRC_VAL(rc2);
9310 if (RT_LIKELY(rc == VINF_SUCCESS))
9311 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9312 else
9313 {
9314 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9315 pVmxTransient->uExitQualification, rc));
9316 }
9317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9318 return rc;
9319}
9320
9321
9322/**
9323 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9324 */
9325HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9326{
9327 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9328 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9329 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9330 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9331 AssertRCReturn(rc, rc);
9332
9333 PVM pVM = pVCpu->CTX_SUFF(pVM);
9334 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9335 if (RT_LIKELY(rc == VINF_SUCCESS))
9336 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9337 else
9338 {
9339 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9340 rc = VERR_EM_INTERPRETER;
9341 }
9342 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9343 return rc;
9344}
9345
9346
9347/**
9348 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9349 */
9350HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9351{
9352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9353 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9354 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9355 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9356 AssertRCReturn(rc, rc);
9357
9358 PVM pVM = pVCpu->CTX_SUFF(pVM);
9359 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9360 rc = VBOXSTRICTRC_VAL(rc2);
9361 if (RT_LIKELY( rc == VINF_SUCCESS
9362 || rc == VINF_EM_HALT))
9363 {
9364 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9365 AssertRCReturn(rc3, rc3);
9366
9367 if ( rc == VINF_EM_HALT
9368 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9369 {
9370 rc = VINF_SUCCESS;
9371 }
9372 }
9373 else
9374 {
9375 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9376 rc = VERR_EM_INTERPRETER;
9377 }
9378 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9379 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9380 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9381 return rc;
9382}
9383
9384
9385/**
9386 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9387 */
9388HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9389{
9390 /*
9391 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9392 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9393 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9394 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9395 */
9396 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9397 HMVMX_RETURN_UNEXPECTED_EXIT();
9398}
9399
9400
9401/**
9402 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9403 */
9404HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9405{
9406 /*
9407 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9408 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9409 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9410 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9411 */
9412 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9413 HMVMX_RETURN_UNEXPECTED_EXIT();
9414}
9415
9416
9417/**
9418 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9419 */
9420HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9421{
9422 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9423 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9424 HMVMX_RETURN_UNEXPECTED_EXIT();
9425}
9426
9427
9428/**
9429 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9430 */
9431HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9432{
9433 /*
9434 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9435 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9436 * See Intel spec. 25.3 "Other Causes of VM-exits".
9437 */
9438 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9439 HMVMX_RETURN_UNEXPECTED_EXIT();
9440}
9441
9442
9443/**
9444 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9445 * VM-exit.
9446 */
9447HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9448{
9449 /*
9450 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9451 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9452 *
9453 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9454 * See Intel spec. "23.8 Restrictions on VMX operation".
9455 */
9456 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9457 return VINF_SUCCESS;
9458}
9459
9460
9461/**
9462 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9463 * VM-exit.
9464 */
9465HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9466{
9467 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9468 return VINF_EM_RESET;
9469}
9470
9471
9472/**
9473 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9474 */
9475HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9476{
9477 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9478 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9479 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9480 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9481 AssertRCReturn(rc, rc);
9482
9483 pMixedCtx->rip++;
9484 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9485 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9486 rc = VINF_SUCCESS;
9487 else
9488 rc = VINF_EM_HALT;
9489
9490 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9491 return rc;
9492}
9493
9494
9495/**
9496 * VM-exit handler for instructions that result in a #UD exception delivered to
9497 * the guest.
9498 */
9499HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9500{
9501 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9502 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9503 return VINF_SUCCESS;
9504}
9505
9506
9507/**
9508 * VM-exit handler for expiry of the VMX preemption timer.
9509 */
9510HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9511{
9512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9513
9514 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9515 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9516
9517 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9518 PVM pVM = pVCpu->CTX_SUFF(pVM);
9519 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9521 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9522}
9523
9524
9525/**
9526 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9527 */
9528HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9529{
9530 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9531
9532 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9533 /** @todo check if XSETBV is supported by the recompiler. */
9534 return VERR_EM_INTERPRETER;
9535}
9536
9537
9538/**
9539 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9540 */
9541HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9542{
9543 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9544
9545 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9546 /** @todo implement EMInterpretInvpcid() */
9547 return VERR_EM_INTERPRETER;
9548}
9549
9550
9551/**
9552 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9553 * Error VM-exit.
9554 */
9555HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9556{
9557 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9558 AssertRCReturn(rc, rc);
9559
9560 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9561 NOREF(uInvalidReason);
9562
9563#ifdef VBOX_STRICT
9564 uint32_t uIntrState;
9565 HMVMXHCUINTREG uHCReg;
9566 uint64_t u64Val;
9567 uint32_t u32Val;
9568
9569 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9570 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9571 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9572 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9573 AssertRCReturn(rc, rc);
9574
9575 Log4(("uInvalidReason %u\n", uInvalidReason));
9576 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9577 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9578 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9579 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9580
9581 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9582 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9583 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9584 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9585 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9586 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9587 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9588 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9589 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9590 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9591 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9592 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9593#endif
9594
9595 PVM pVM = pVCpu->CTX_SUFF(pVM);
9596 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9597
9598 return VERR_VMX_INVALID_GUEST_STATE;
9599}
9600
9601
9602/**
9603 * VM-exit handler for VM-entry failure due to an MSR-load
9604 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9605 */
9606HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9607{
9608 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9609 HMVMX_RETURN_UNEXPECTED_EXIT();
9610}
9611
9612
9613/**
9614 * VM-exit handler for VM-entry failure due to a machine-check event
9615 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9616 */
9617HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9618{
9619 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9620 HMVMX_RETURN_UNEXPECTED_EXIT();
9621}
9622
9623
9624/**
9625 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9626 * theory.
9627 */
9628HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9629{
9630 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9631 return VERR_VMX_UNDEFINED_EXIT_CODE;
9632}
9633
9634
9635/**
9636 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9637 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9638 * Conditional VM-exit.
9639 */
9640HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9641{
9642 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9643
9644 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9646 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9647 return VERR_EM_INTERPRETER;
9648 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9649 HMVMX_RETURN_UNEXPECTED_EXIT();
9650}
9651
9652
9653/**
9654 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9655 */
9656HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9657{
9658 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9659
9660 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9661 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9662 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9663 return VERR_EM_INTERPRETER;
9664 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9665 HMVMX_RETURN_UNEXPECTED_EXIT();
9666}
9667
9668
9669/**
9670 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9671 */
9672HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9673{
9674 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9675
9676 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9677 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9678 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9679 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9680 AssertRCReturn(rc, rc);
9681 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9682
9683 PVM pVM = pVCpu->CTX_SUFF(pVM);
9684 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9685 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9686 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9687 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9688
9689 if (RT_LIKELY(rc == VINF_SUCCESS))
9690 {
9691 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9692 Assert(pVmxTransient->cbInstr == 2);
9693 }
9694 return rc;
9695}
9696
9697
9698/**
9699 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9700 */
9701HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9702{
9703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9704 PVM pVM = pVCpu->CTX_SUFF(pVM);
9705 int rc = VINF_SUCCESS;
9706
9707 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9708 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9709 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9710 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9711 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9712 AssertRCReturn(rc, rc);
9713 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9714
9715 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9716 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9718
9719 if (RT_LIKELY(rc == VINF_SUCCESS))
9720 {
9721 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9722
9723 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9724 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9725 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9726 {
9727 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9728 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9729 EMInterpretWrmsr() changes it. */
9730 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9731 }
9732 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9733 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9734 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9735 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9736
9737 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9738 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9739 {
9740 switch (pMixedCtx->ecx)
9741 {
9742 case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9743 case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9744 case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9745 case MSR_K8_FS_BASE: /* no break */
9746 case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9747 case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9748 }
9749 }
9750#ifdef VBOX_STRICT
9751 else
9752 {
9753 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9754 switch (pMixedCtx->ecx)
9755 {
9756 case MSR_IA32_SYSENTER_CS:
9757 case MSR_IA32_SYSENTER_EIP:
9758 case MSR_IA32_SYSENTER_ESP:
9759 case MSR_K8_FS_BASE:
9760 case MSR_K8_GS_BASE:
9761 {
9762 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9763 HMVMX_RETURN_UNEXPECTED_EXIT();
9764 }
9765
9766 case MSR_K8_LSTAR:
9767 case MSR_K6_STAR:
9768 case MSR_K8_SF_MASK:
9769 case MSR_K8_TSC_AUX:
9770 case MSR_K8_KERNEL_GS_BASE:
9771 {
9772 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9773 pMixedCtx->ecx));
9774 HMVMX_RETURN_UNEXPECTED_EXIT();
9775 }
9776 }
9777 }
9778#endif /* VBOX_STRICT */
9779 }
9780 return rc;
9781}
9782
9783
9784/**
9785 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9786 */
9787HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9788{
9789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9790
9791 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9792 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9793 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9794 return VERR_EM_INTERPRETER;
9795 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9796 HMVMX_RETURN_UNEXPECTED_EXIT();
9797}
9798
9799
9800/**
9801 * VM-exit handler for when the TPR value is lowered below the specified
9802 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9803 */
9804HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9805{
9806 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9807 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9808
9809 /*
9810 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9811 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9812 * resume guest execution.
9813 */
9814 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9815 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9816 return VINF_SUCCESS;
9817}
9818
9819
9820/**
9821 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9822 * VM-exit.
9823 *
9824 * @retval VINF_SUCCESS when guest execution can continue.
9825 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9826 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9827 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9828 * recompiler.
9829 */
9830HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9831{
9832 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9833 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9834 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9835 AssertRCReturn(rc, rc);
9836
9837 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9838 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9839 PVM pVM = pVCpu->CTX_SUFF(pVM);
9840 switch (uAccessType)
9841 {
9842 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9843 {
9844#if 0
9845 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9846 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9847#else
9848 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9849 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9850 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9851#endif
9852 AssertRCReturn(rc, rc);
9853
9854 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9855 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9856 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9857 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9858
9859 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9860 {
9861 case 0: /* CR0 */
9862 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9863 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9864 break;
9865 case 2: /* CR2 */
9866 /* Nothing to do here, CR2 it's not part of the VMCS. */
9867 break;
9868 case 3: /* CR3 */
9869 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9870 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9871 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9872 break;
9873 case 4: /* CR4 */
9874 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9875 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9876 break;
9877 case 8: /* CR8 */
9878 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9879 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9880 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9881 break;
9882 default:
9883 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9884 break;
9885 }
9886
9887 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9888 break;
9889 }
9890
9891 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9892 {
9893 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9894 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9895 AssertRCReturn(rc, rc);
9896 Assert( !pVM->hm.s.fNestedPaging
9897 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9898 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9899
9900 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9901 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9902 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9903
9904 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9905 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9906 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9907 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9908 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9909 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9910 break;
9911 }
9912
9913 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9914 {
9915 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9916 AssertRCReturn(rc, rc);
9917 rc = EMInterpretCLTS(pVM, pVCpu);
9918 AssertRCReturn(rc, rc);
9919 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9920 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9921 Log4(("CRX CLTS write rc=%d\n", rc));
9922 break;
9923 }
9924
9925 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9926 {
9927 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9928 AssertRCReturn(rc, rc);
9929 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9930 if (RT_LIKELY(rc == VINF_SUCCESS))
9931 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9932 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9933 Log4(("CRX LMSW write rc=%d\n", rc));
9934 break;
9935 }
9936
9937 default:
9938 {
9939 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9940 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9941 }
9942 }
9943
9944 /* Validate possible error codes. */
9945 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9946 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9947 if (RT_SUCCESS(rc))
9948 {
9949 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9950 AssertRCReturn(rc2, rc2);
9951 }
9952
9953 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9954 return rc;
9955}
9956
9957
9958/**
9959 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9960 * VM-exit.
9961 */
9962HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9963{
9964 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9965 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9966
9967 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9968 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9969 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9970 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9971 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9972 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9973 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9974 AssertRCReturn(rc2, rc2);
9975
9976 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9977 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9978 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9979 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9980 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9981 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9982 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9983
9984 /* I/O operation lookup arrays. */
9985 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9986 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9987
9988 VBOXSTRICTRC rcStrict;
9989 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9990 const uint32_t cbInstr = pVmxTransient->cbInstr;
9991 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9992 PVM pVM = pVCpu->CTX_SUFF(pVM);
9993 if (fIOString)
9994 {
9995#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9996 /*
9997 * INS/OUTS - I/O String instruction.
9998 *
9999 * Use instruction-information if available, otherwise fall back on
10000 * interpreting the instruction.
10001 */
10002 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10003 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10004 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10005 {
10006 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
10007 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10008 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10009 AssertRCReturn(rc2, rc2);
10010 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10011 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10012 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10013 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10014 if (fIOWrite)
10015 {
10016 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10017 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10018 }
10019 else
10020 {
10021 /*
10022 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10023 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10024 * See Intel Instruction spec. for "INS".
10025 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10026 */
10027 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10028 }
10029 }
10030 else
10031 {
10032 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10033 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10034 AssertRCReturn(rc2, rc2);
10035 rcStrict = IEMExecOne(pVCpu);
10036 }
10037 /** @todo IEM needs to be setting these flags somehow. */
10038 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10039 fUpdateRipAlready = true;
10040#else
10041 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10042 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
10043 if (RT_SUCCESS(rcStrict))
10044 {
10045 if (fIOWrite)
10046 {
10047 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10048 (DISCPUMODE)pDis->uAddrMode, cbValue);
10049 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10050 }
10051 else
10052 {
10053 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10054 (DISCPUMODE)pDis->uAddrMode, cbValue);
10055 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10056 }
10057 }
10058 else
10059 {
10060 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10061 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10062 }
10063#endif
10064 }
10065 else
10066 {
10067 /*
10068 * IN/OUT - I/O instruction.
10069 */
10070 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10071 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10072 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10073 if (fIOWrite)
10074 {
10075 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10076 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10077 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10078 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10079 }
10080 else
10081 {
10082 uint32_t u32Result = 0;
10083 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10084 if (IOM_SUCCESS(rcStrict))
10085 {
10086 /* Save result of I/O IN instr. in AL/AX/EAX. */
10087 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10088 }
10089 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10090 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10091 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10092 }
10093 }
10094
10095 if (IOM_SUCCESS(rcStrict))
10096 {
10097 if (!fUpdateRipAlready)
10098 {
10099 pMixedCtx->rip += cbInstr;
10100 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10101 }
10102
10103 /* INS & OUTS with REP prefix modify RFLAGS. */
10104 if (fIOString)
10105 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10106
10107 /*
10108 * If any I/O breakpoints are armed, we need to check if one triggered
10109 * and take appropriate action.
10110 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10111 */
10112 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10113 AssertRCReturn(rc2, rc2);
10114
10115 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10116 * execution engines about whether hyper BPs and such are pending. */
10117 uint32_t const uDr7 = pMixedCtx->dr[7];
10118 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10119 && X86_DR7_ANY_RW_IO(uDr7)
10120 && (pMixedCtx->cr4 & X86_CR4_DE))
10121 || DBGFBpIsHwIoArmed(pVM)))
10122 {
10123 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10124
10125 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10126 VMMRZCallRing3Disable(pVCpu);
10127 HM_DISABLE_PREEMPT_IF_NEEDED();
10128
10129 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10130
10131 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10132 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10133 {
10134 /* Raise #DB. */
10135 if (fIsGuestDbgActive)
10136 ASMSetDR6(pMixedCtx->dr[6]);
10137 if (pMixedCtx->dr[7] != uDr7)
10138 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10139
10140 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10141 }
10142 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10143 else if ( rcStrict2 != VINF_SUCCESS
10144 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10145 rcStrict = rcStrict2;
10146
10147 HM_RESTORE_PREEMPT_IF_NEEDED();
10148 VMMRZCallRing3Enable(pVCpu);
10149 }
10150 }
10151
10152#ifdef DEBUG
10153 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10154 Assert(!fIOWrite);
10155 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10156 Assert(fIOWrite);
10157 else
10158 {
10159 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10160 * statuses, that the VMM device and some others may return. See
10161 * IOM_SUCCESS() for guidance. */
10162 AssertMsg( RT_FAILURE(rcStrict)
10163 || rcStrict == VINF_SUCCESS
10164 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10165 || rcStrict == VINF_EM_DBG_BREAKPOINT
10166 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10167 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10168 }
10169#endif
10170
10171 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10172 return VBOXSTRICTRC_TODO(rcStrict);
10173}
10174
10175
10176/**
10177 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10178 * VM-exit.
10179 */
10180HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10181{
10182 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10183
10184 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10185 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10186 AssertRCReturn(rc, rc);
10187 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10188 {
10189 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10190 AssertRCReturn(rc, rc);
10191 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10192 {
10193 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10194
10195 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10196 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10197 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10198 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10199 {
10200 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10201 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10202
10203 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10204 Assert(!pVCpu->hm.s.Event.fPending);
10205 pVCpu->hm.s.Event.fPending = true;
10206 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10207 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10208 AssertRCReturn(rc, rc);
10209 if (fErrorCodeValid)
10210 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10211 else
10212 pVCpu->hm.s.Event.u32ErrCode = 0;
10213 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10214 && uVector == X86_XCPT_PF)
10215 {
10216 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10217 }
10218
10219 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10220 }
10221 }
10222 }
10223
10224 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10225 * emulation. */
10226 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10227 return VERR_EM_INTERPRETER;
10228}
10229
10230
10231/**
10232 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10233 */
10234HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10235{
10236 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10237 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10238 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10239 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10240 AssertRCReturn(rc, rc);
10241 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10242 return VINF_EM_DBG_STEPPED;
10243}
10244
10245
10246/**
10247 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10248 */
10249HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10250{
10251 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10252
10253 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10254 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10255 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10256 return VINF_SUCCESS;
10257 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10258 return rc;
10259
10260#if 0
10261 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10262 * just sync the whole thing. */
10263 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10264#else
10265 /* Aggressive state sync. for now. */
10266 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10267 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10268 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10269#endif
10270 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10271 AssertRCReturn(rc, rc);
10272
10273 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10274 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10275 switch (uAccessType)
10276 {
10277 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10278 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10279 {
10280 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10281 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10282 {
10283 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10284 }
10285
10286 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10287 GCPhys &= PAGE_BASE_GC_MASK;
10288 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10289 PVM pVM = pVCpu->CTX_SUFF(pVM);
10290 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10291 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10292
10293 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10294 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10295 CPUMCTX2CORE(pMixedCtx), GCPhys);
10296 rc = VBOXSTRICTRC_VAL(rc2);
10297 Log4(("ApicAccess rc=%d\n", rc));
10298 if ( rc == VINF_SUCCESS
10299 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10300 || rc == VERR_PAGE_NOT_PRESENT)
10301 {
10302 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10303 | HM_CHANGED_GUEST_RSP
10304 | HM_CHANGED_GUEST_RFLAGS
10305 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10306 rc = VINF_SUCCESS;
10307 }
10308 break;
10309 }
10310
10311 default:
10312 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10313 rc = VINF_EM_RAW_EMULATE_INSTR;
10314 break;
10315 }
10316
10317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10318 return rc;
10319}
10320
10321
10322/**
10323 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10324 * VM-exit.
10325 */
10326HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10327{
10328 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10329
10330 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10331 if (pVmxTransient->fWasGuestDebugStateActive)
10332 {
10333 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10334 HMVMX_RETURN_UNEXPECTED_EXIT();
10335 }
10336
10337 int rc = VERR_INTERNAL_ERROR_5;
10338 if ( !DBGFIsStepping(pVCpu)
10339 && !pVCpu->hm.s.fSingleInstruction
10340 && !pVmxTransient->fWasHyperDebugStateActive)
10341 {
10342 /* Don't intercept MOV DRx and #DB any more. */
10343 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10344 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10345 AssertRCReturn(rc, rc);
10346
10347 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10348 {
10349#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10350 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10351 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10352 AssertRCReturn(rc, rc);
10353#endif
10354 }
10355
10356 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10357 VMMRZCallRing3Disable(pVCpu);
10358 HM_DISABLE_PREEMPT_IF_NEEDED();
10359
10360 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10361 PVM pVM = pVCpu->CTX_SUFF(pVM);
10362 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10363 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10364
10365 HM_RESTORE_PREEMPT_IF_NEEDED();
10366 VMMRZCallRing3Enable(pVCpu);
10367
10368#ifdef VBOX_WITH_STATISTICS
10369 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10370 AssertRCReturn(rc, rc);
10371 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10372 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10373 else
10374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10375#endif
10376 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10377 return VINF_SUCCESS;
10378 }
10379
10380 /*
10381 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10382 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10383 */
10384 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10385 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10386 AssertRCReturn(rc, rc);
10387 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10388
10389 PVM pVM = pVCpu->CTX_SUFF(pVM);
10390 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10391 {
10392 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10393 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10394 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10395 if (RT_SUCCESS(rc))
10396 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10397 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10398 }
10399 else
10400 {
10401 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10402 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10403 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10404 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10405 }
10406
10407 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10408 if (RT_SUCCESS(rc))
10409 {
10410 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10411 AssertRCReturn(rc2, rc2);
10412 }
10413 return rc;
10414}
10415
10416
10417/**
10418 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10419 * Conditional VM-exit.
10420 */
10421HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10422{
10423 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10424 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10425
10426 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10427 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10428 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10429 return VINF_SUCCESS;
10430 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10431 return rc;
10432
10433 RTGCPHYS GCPhys = 0;
10434 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10435
10436#if 0
10437 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10438#else
10439 /* Aggressive state sync. for now. */
10440 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10441 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10442 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10443#endif
10444 AssertRCReturn(rc, rc);
10445
10446 /*
10447 * If we succeed, resume guest execution.
10448 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10449 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10450 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10451 * weird case. See @bugref{6043}.
10452 */
10453 PVM pVM = pVCpu->CTX_SUFF(pVM);
10454 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10455 rc = VBOXSTRICTRC_VAL(rc2);
10456 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10457 if ( rc == VINF_SUCCESS
10458 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10459 || rc == VERR_PAGE_NOT_PRESENT)
10460 {
10461 /* Successfully handled MMIO operation. */
10462 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10463 | HM_CHANGED_GUEST_RSP
10464 | HM_CHANGED_GUEST_RFLAGS
10465 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10466 rc = VINF_SUCCESS;
10467 }
10468 return rc;
10469}
10470
10471
10472/**
10473 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10474 * VM-exit.
10475 */
10476HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10477{
10478 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10479 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10480
10481 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10482 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10483 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10484 return VINF_SUCCESS;
10485 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10486 return rc;
10487
10488 RTGCPHYS GCPhys = 0;
10489 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10490 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10491#if 0
10492 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10493#else
10494 /* Aggressive state sync. for now. */
10495 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10496 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10497 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10498#endif
10499 AssertRCReturn(rc, rc);
10500
10501 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10502 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10503
10504 RTGCUINT uErrorCode = 0;
10505 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10506 uErrorCode |= X86_TRAP_PF_ID;
10507 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10508 uErrorCode |= X86_TRAP_PF_RW;
10509 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10510 uErrorCode |= X86_TRAP_PF_P;
10511
10512 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10513
10514 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10515 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10516
10517 /* Handle the pagefault trap for the nested shadow table. */
10518 PVM pVM = pVCpu->CTX_SUFF(pVM);
10519 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10520 TRPMResetTrap(pVCpu);
10521
10522 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10523 if ( rc == VINF_SUCCESS
10524 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10525 || rc == VERR_PAGE_NOT_PRESENT)
10526 {
10527 /* Successfully synced our nested page tables. */
10528 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10529 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10530 | HM_CHANGED_GUEST_RSP
10531 | HM_CHANGED_GUEST_RFLAGS);
10532 return VINF_SUCCESS;
10533 }
10534
10535 Log4(("EPT return to ring-3 rc=%d\n"));
10536 return rc;
10537}
10538
10539/** @} */
10540
10541/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10542/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10543/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10544
10545/** @name VM-exit exception handlers.
10546 * @{
10547 */
10548
10549/**
10550 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10551 */
10552static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10553{
10554 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10555 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10556
10557 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10558 AssertRCReturn(rc, rc);
10559
10560 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10561 {
10562 /* Old-style FPU error reporting needs some extra work. */
10563 /** @todo don't fall back to the recompiler, but do it manually. */
10564 return VERR_EM_INTERPRETER;
10565 }
10566
10567 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10568 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10569 return rc;
10570}
10571
10572
10573/**
10574 * VM-exit exception handler for #BP (Breakpoint exception).
10575 */
10576static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10577{
10578 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10579 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10580
10581 /** @todo Try optimize this by not saving the entire guest state unless
10582 * really needed. */
10583 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10584 AssertRCReturn(rc, rc);
10585
10586 PVM pVM = pVCpu->CTX_SUFF(pVM);
10587 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10588 if (rc == VINF_EM_RAW_GUEST_TRAP)
10589 {
10590 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10591 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10592 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10593 AssertRCReturn(rc, rc);
10594
10595 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10596 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10597 }
10598
10599 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10600 return rc;
10601}
10602
10603
10604/**
10605 * VM-exit exception handler for #DB (Debug exception).
10606 */
10607static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10608{
10609 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10610 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10611 Log6(("XcptDB\n"));
10612
10613 /*
10614 * Get the DR6-like values from the exit qualification and pass it to DBGF
10615 * for processing.
10616 */
10617 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10618 AssertRCReturn(rc, rc);
10619
10620 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10621 uint64_t uDR6 = X86_DR6_INIT_VAL;
10622 uDR6 |= ( pVmxTransient->uExitQualification
10623 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10624
10625 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10626 if (rc == VINF_EM_RAW_GUEST_TRAP)
10627 {
10628 /*
10629 * The exception was for the guest. Update DR6, DR7.GD and
10630 * IA32_DEBUGCTL.LBR before forwarding it.
10631 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10632 */
10633 VMMRZCallRing3Disable(pVCpu);
10634 HM_DISABLE_PREEMPT_IF_NEEDED();
10635
10636 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10637 pMixedCtx->dr[6] |= uDR6;
10638 if (CPUMIsGuestDebugStateActive(pVCpu))
10639 ASMSetDR6(pMixedCtx->dr[6]);
10640
10641 HM_RESTORE_PREEMPT_IF_NEEDED();
10642 VMMRZCallRing3Enable(pVCpu);
10643
10644 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10645 AssertRCReturn(rc, rc);
10646
10647 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10648 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10649
10650 /* Paranoia. */
10651 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10652 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10653
10654 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10655 AssertRCReturn(rc, rc);
10656
10657 /*
10658 * Raise #DB in the guest.
10659 */
10660 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10661 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10662 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10663 AssertRCReturn(rc, rc);
10664 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10665 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10666 return VINF_SUCCESS;
10667 }
10668
10669 /*
10670 * Not a guest trap, must be a hypervisor related debug event then.
10671 * Update DR6 in case someone is interested in it.
10672 */
10673 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10674 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10675 CPUMSetHyperDR6(pVCpu, uDR6);
10676
10677 return rc;
10678}
10679
10680
10681/**
10682 * VM-exit exception handler for #NM (Device-not-available exception: floating
10683 * point exception).
10684 */
10685static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10686{
10687 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10688
10689 /* We require CR0 and EFER. EFER is always up-to-date. */
10690 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10691 AssertRCReturn(rc, rc);
10692
10693 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10694 VMMRZCallRing3Disable(pVCpu);
10695 HM_DISABLE_PREEMPT_IF_NEEDED();
10696
10697 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10698 if (pVmxTransient->fWasGuestFPUStateActive)
10699 {
10700 rc = VINF_EM_RAW_GUEST_TRAP;
10701 Assert(CPUMIsGuestFPUStateActive(pVCpu) || VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10702 }
10703 else
10704 {
10705#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10706 Assert(!pVmxTransient->fWasGuestFPUStateActive);
10707#endif
10708 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10709 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10710 }
10711
10712 HM_RESTORE_PREEMPT_IF_NEEDED();
10713 VMMRZCallRing3Enable(pVCpu);
10714
10715 if (rc == VINF_SUCCESS)
10716 {
10717 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10718 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10719 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10720 pVCpu->hm.s.fUseGuestFpu = true;
10721 }
10722 else
10723 {
10724 /* Forward #NM to the guest. */
10725 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10726 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10727 AssertRCReturn(rc, rc);
10728 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10729 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10730 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10731 }
10732
10733 return VINF_SUCCESS;
10734}
10735
10736
10737/**
10738 * VM-exit exception handler for #GP (General-protection exception).
10739 *
10740 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10741 */
10742static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10743{
10744 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10745 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10746
10747 int rc = VERR_INTERNAL_ERROR_5;
10748 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10749 {
10750#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10751 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10752 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10753 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10754 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10755 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10756 AssertRCReturn(rc, rc);
10757 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10758 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10759 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10760 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10761 return rc;
10762#else
10763 /* We don't intercept #GP. */
10764 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10765 return VERR_VMX_UNEXPECTED_EXCEPTION;
10766#endif
10767 }
10768
10769 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10770 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10771
10772 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10773 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10774 AssertRCReturn(rc, rc);
10775
10776 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10777 uint32_t cbOp = 0;
10778 PVM pVM = pVCpu->CTX_SUFF(pVM);
10779 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10780 if (RT_SUCCESS(rc))
10781 {
10782 rc = VINF_SUCCESS;
10783 Assert(cbOp == pDis->cbInstr);
10784 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10785 switch (pDis->pCurInstr->uOpcode)
10786 {
10787 case OP_CLI:
10788 {
10789 pMixedCtx->eflags.Bits.u1IF = 0;
10790 pMixedCtx->rip += pDis->cbInstr;
10791 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10792 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10793 break;
10794 }
10795
10796 case OP_STI:
10797 {
10798 pMixedCtx->eflags.Bits.u1IF = 1;
10799 pMixedCtx->rip += pDis->cbInstr;
10800 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10801 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10802 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10804 break;
10805 }
10806
10807 case OP_HLT:
10808 {
10809 rc = VINF_EM_HALT;
10810 pMixedCtx->rip += pDis->cbInstr;
10811 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10813 break;
10814 }
10815
10816 case OP_POPF:
10817 {
10818 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10819 uint32_t cbParm = 0;
10820 uint32_t uMask = 0;
10821 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10822 {
10823 cbParm = 4;
10824 uMask = 0xffffffff;
10825 }
10826 else
10827 {
10828 cbParm = 2;
10829 uMask = 0xffff;
10830 }
10831
10832 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10833 RTGCPTR GCPtrStack = 0;
10834 X86EFLAGS Eflags;
10835 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10836 &GCPtrStack);
10837 if (RT_SUCCESS(rc))
10838 {
10839 Assert(sizeof(Eflags.u32) >= cbParm);
10840 Eflags.u32 = 0;
10841 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10842 }
10843 if (RT_FAILURE(rc))
10844 {
10845 rc = VERR_EM_INTERPRETER;
10846 break;
10847 }
10848 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10849 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10850 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10851 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10852 pMixedCtx->esp += cbParm;
10853 pMixedCtx->esp &= uMask;
10854 pMixedCtx->rip += pDis->cbInstr;
10855
10856 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10857 | HM_CHANGED_GUEST_RSP
10858 | HM_CHANGED_GUEST_RFLAGS);
10859 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10860 break;
10861 }
10862
10863 case OP_PUSHF:
10864 {
10865 uint32_t cbParm = 0;
10866 uint32_t uMask = 0;
10867 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10868 {
10869 cbParm = 4;
10870 uMask = 0xffffffff;
10871 }
10872 else
10873 {
10874 cbParm = 2;
10875 uMask = 0xffff;
10876 }
10877
10878 /* Get the stack pointer & push the contents of eflags onto the stack. */
10879 RTGCPTR GCPtrStack = 0;
10880 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10881 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10882 if (RT_FAILURE(rc))
10883 {
10884 rc = VERR_EM_INTERPRETER;
10885 break;
10886 }
10887 X86EFLAGS Eflags = pMixedCtx->eflags;
10888 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10889 Eflags.Bits.u1RF = 0;
10890 Eflags.Bits.u1VM = 0;
10891
10892 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10893 if (RT_FAILURE(rc))
10894 {
10895 rc = VERR_EM_INTERPRETER;
10896 break;
10897 }
10898 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10899 pMixedCtx->esp -= cbParm;
10900 pMixedCtx->esp &= uMask;
10901 pMixedCtx->rip += pDis->cbInstr;
10902 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
10903 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10904 break;
10905 }
10906
10907 case OP_IRET:
10908 {
10909 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10910 * instruction reference. */
10911 RTGCPTR GCPtrStack = 0;
10912 uint32_t uMask = 0xffff;
10913 uint16_t aIretFrame[3];
10914 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10915 {
10916 rc = VERR_EM_INTERPRETER;
10917 break;
10918 }
10919 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10920 &GCPtrStack);
10921 if (RT_SUCCESS(rc))
10922 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10923 if (RT_FAILURE(rc))
10924 {
10925 rc = VERR_EM_INTERPRETER;
10926 break;
10927 }
10928 pMixedCtx->eip = 0;
10929 pMixedCtx->ip = aIretFrame[0];
10930 pMixedCtx->cs.Sel = aIretFrame[1];
10931 pMixedCtx->cs.ValidSel = aIretFrame[1];
10932 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10933 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10934 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10935 pMixedCtx->sp += sizeof(aIretFrame);
10936 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10937 | HM_CHANGED_GUEST_SEGMENT_REGS
10938 | HM_CHANGED_GUEST_RSP
10939 | HM_CHANGED_GUEST_RFLAGS);
10940 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10941 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10942 break;
10943 }
10944
10945 case OP_INT:
10946 {
10947 uint16_t uVector = pDis->Param1.uValue & 0xff;
10948 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10950 break;
10951 }
10952
10953 case OP_INTO:
10954 {
10955 if (pMixedCtx->eflags.Bits.u1OF)
10956 {
10957 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10958 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10959 }
10960 break;
10961 }
10962
10963 default:
10964 {
10965 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10966 EMCODETYPE_SUPERVISOR);
10967 rc = VBOXSTRICTRC_VAL(rc2);
10968 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10969 Log4(("#GP rc=%Rrc\n", rc));
10970 break;
10971 }
10972 }
10973 }
10974 else
10975 rc = VERR_EM_INTERPRETER;
10976
10977 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10978 ("#GP Unexpected rc=%Rrc\n", rc));
10979 return rc;
10980}
10981
10982
10983/**
10984 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10985 * the exception reported in the VMX transient structure back into the VM.
10986 *
10987 * @remarks Requires uExitIntInfo in the VMX transient structure to be
10988 * up-to-date.
10989 */
10990static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10991{
10992 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10993
10994 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10995 hmR0VmxCheckExitDueToEventDelivery(). */
10996 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10997 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10998 AssertRCReturn(rc, rc);
10999 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11000
11001 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11002 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11003 return VINF_SUCCESS;
11004}
11005
11006
11007/**
11008 * VM-exit exception handler for #PF (Page-fault exception).
11009 */
11010static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11011{
11012 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11013 PVM pVM = pVCpu->CTX_SUFF(pVM);
11014 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11015 rc |= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
11016 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11017 AssertRCReturn(rc, rc);
11018
11019#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11020 if (pVM->hm.s.fNestedPaging)
11021 {
11022 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11023 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11024 {
11025 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11026 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11027 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11028 }
11029 else
11030 {
11031 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11032 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11033 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11034 }
11035 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11036 return rc;
11037 }
11038#else
11039 Assert(!pVM->hm.s.fNestedPaging);
11040#endif
11041
11042 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11043 AssertRCReturn(rc, rc);
11044
11045 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11046 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11047
11048 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11049 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11050 (RTGCPTR)pVmxTransient->uExitQualification);
11051
11052 Log4(("#PF: rc=%Rrc\n", rc));
11053 if (rc == VINF_SUCCESS)
11054 {
11055 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11056 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11057 * memory? We don't update the whole state here... */
11058 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11059 | HM_CHANGED_GUEST_RSP
11060 | HM_CHANGED_GUEST_RFLAGS
11061 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11062 TRPMResetTrap(pVCpu);
11063 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11064 return rc;
11065 }
11066 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11067 {
11068 if (!pVmxTransient->fVectoringPF)
11069 {
11070 /* It's a guest page fault and needs to be reflected to the guest. */
11071 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11072 TRPMResetTrap(pVCpu);
11073 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11074 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11075 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11076 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11077 }
11078 else
11079 {
11080 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11081 TRPMResetTrap(pVCpu);
11082 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11083 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11084 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11085 }
11086
11087 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11088 return VINF_SUCCESS;
11089 }
11090
11091 TRPMResetTrap(pVCpu);
11092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11093 return rc;
11094}
11095
11096/** @} */
11097
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette