VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49285

Last change on this file since 49285 was 49285, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: nits.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 461.4 KB
Line 
1/* $Id: HMVMXR0.cpp 49285 2013-10-25 09:50:20Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#define HMVMX_ALWAYS_FLUSH_TLB
46#endif
47
48
49/*******************************************************************************
50* Defined Constants And Macros *
51*******************************************************************************/
52#if defined(RT_ARCH_AMD64)
53# define HMVMX_IS_64BIT_HOST_MODE() (true)
54typedef RTHCUINTREG HMVMXHCUINTREG;
55#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56extern "C" uint32_t g_fVMXIs64bitHost;
57# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58typedef uint64_t HMVMXHCUINTREG;
59#else
60# define HMVMX_IS_64BIT_HOST_MODE() (false)
61typedef RTHCUINTREG HMVMXHCUINTREG;
62#endif
63
64/** Use the function table. */
65#define HMVMX_USE_FUNCTION_TABLE
66
67/**
68 * The maximum number of MSRs we are willing to swap during a world-switch.
69 * Intel claims 512/check capability MSR, we don't want to do anywhere close
70 * to that. See Intel spec. 24.7.2 "VM-Exit Controls for MSRs"
71 *
72 * Bump this count as and when required, there's no backward compatibility
73 * requirement.
74 */
75#define HMVMX_MAX_SWAP_MSR_COUNT 5
76
77/** Determine which tagged-TLB flush handler to use. */
78#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
79#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
80#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
81#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
82
83/** @name Updated-guest-state flags.
84 * @{ */
85#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
86#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
87#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
88#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
89#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
90#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
91#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
92#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
93#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
94#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
95#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
96#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
97#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
98#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
99#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
100#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
101#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
102#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
103#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
104#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
105#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
106 | HMVMX_UPDATED_GUEST_RSP \
107 | HMVMX_UPDATED_GUEST_RFLAGS \
108 | HMVMX_UPDATED_GUEST_CR0 \
109 | HMVMX_UPDATED_GUEST_CR3 \
110 | HMVMX_UPDATED_GUEST_CR4 \
111 | HMVMX_UPDATED_GUEST_GDTR \
112 | HMVMX_UPDATED_GUEST_IDTR \
113 | HMVMX_UPDATED_GUEST_LDTR \
114 | HMVMX_UPDATED_GUEST_TR \
115 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
116 | HMVMX_UPDATED_GUEST_DEBUG \
117 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
118 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
119 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
120 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
121 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
122 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
123 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
124 | HMVMX_UPDATED_GUEST_APIC_STATE)
125/** @} */
126
127/** @name
128 * Flags to skip redundant reads of some common VMCS fields that are not part of
129 * the guest-CPU state but are in the transient structure.
130 */
131#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
132#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
133#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
134#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
135#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
136#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
137#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
138/** @} */
139
140/** @name
141 * States of the VMCS.
142 *
143 * This does not reflect all possible VMCS states but currently only those
144 * needed for maintaining the VMCS consistently even when thread-context hooks
145 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
146 */
147#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
148#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
149#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
150/** @} */
151
152/**
153 * Exception bitmap mask for real-mode guests (real-on-v86).
154 *
155 * We need to intercept all exceptions manually (except #PF). #NM is also
156 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
157 * even in real-mode if we have Nested Paging support.
158 */
159#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
160 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
161 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
162 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
163 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
164 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
165 | RT_BIT(X86_XCPT_XF))
166
167/**
168 * Exception bitmap mask for all contributory exceptions.
169 *
170 * Page fault is deliberately excluded here as it's conditional as to whether
171 * it's contributory or benign. Page faults are handled separately.
172 */
173#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
174 | RT_BIT(X86_XCPT_DE))
175
176/** Maximum VM-instruction error number. */
177#define HMVMX_INSTR_ERROR_MAX 28
178
179/** Profiling macro. */
180#ifdef HM_PROFILE_EXIT_DISPATCH
181# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
183#else
184# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
185# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
186#endif
187
188/** Assert that preemption is disabled or covered by thread-context hooks. */
189#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
191
192/** Assert that we haven't migrated CPUs when thread-context hooks are not
193 * used. */
194#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
195 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
196 ("Illegal migration! Entered on CPU %u Current %u\n", \
197 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
198
199/** Helper macro for VM-exit handlers called unexpectedly. */
200#define HMVMX_RETURN_UNEXPECTED_EXIT() \
201 do { \
202 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
203 return VERR_VMX_UNEXPECTED_EXIT; \
204 } while (0)
205
206
207/*******************************************************************************
208* Structures and Typedefs *
209*******************************************************************************/
210/**
211 * VMX transient state.
212 *
213 * A state structure for holding miscellaneous information across
214 * VMX non-root operation and restored after the transition.
215 */
216typedef struct VMXTRANSIENT
217{
218 /** The host's rflags/eflags. */
219 RTCCUINTREG uEflags;
220#if HC_ARCH_BITS == 32
221 uint32_t u32Alignment0;
222#endif
223 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
224 uint64_t u64LStarMsr;
225 /** The guest's TPR value used for TPR shadowing. */
226 uint8_t u8GuestTpr;
227 /** Alignment. */
228 uint8_t abAlignment0[7];
229
230 /** The basic VM-exit reason. */
231 uint16_t uExitReason;
232 /** Alignment. */
233 uint16_t u16Alignment0;
234 /** The VM-exit interruption error code. */
235 uint32_t uExitIntErrorCode;
236 /** The VM-exit exit qualification. */
237 uint64_t uExitQualification;
238
239 /** The VM-exit interruption-information field. */
240 uint32_t uExitIntInfo;
241 /** The VM-exit instruction-length field. */
242 uint32_t cbInstr;
243 /** The VM-exit instruction-information field. */
244 union
245 {
246 /** Plain unsigned int representation. */
247 uint32_t u;
248 /** INS and OUTS information. */
249 struct
250 {
251 uint32_t u6Reserved0 : 7;
252 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253 uint32_t u3AddrSize : 3;
254 uint32_t u5Reserved1 : 5;
255 /** The segment register (X86_SREG_XXX). */
256 uint32_t iSegReg : 3;
257 uint32_t uReserved2 : 14;
258 } StrIo;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest FPU was active at the time of VM-exit. */
281 bool fWasGuestFPUStateActive;
282 /** Whether the guest debug state was active at the time of VM-exit. */
283 bool fWasGuestDebugStateActive;
284 /** Whether the hyper debug state was active at the time of VM-exit. */
285 bool fWasHyperDebugStateActive;
286 /** Whether TSC-offsetting should be setup before VM-entry. */
287 bool fUpdateTscOffsettingAndPreemptTimer;
288 /** Whether the VM-exit was caused by a page-fault during delivery of a
289 * contributory exception or a page-fault. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322
323/**
324 * VMX VM-exit handler.
325 *
326 * @returns VBox status code.
327 * @param pVCpu Pointer to the VMCPU.
328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
329 * out-of-sync. Make sure to update the required
330 * fields before using them.
331 * @param pVmxTransient Pointer to the VMX-transient structure.
332 */
333#ifndef HMVMX_USE_FUNCTION_TABLE
334typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335#else
336typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
337/** Pointer to VM-exit handler. */
338typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
339#endif
340
341
342/*******************************************************************************
343* Internal Functions *
344*******************************************************************************/
345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
346static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
347static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
348static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
350#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
351static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
352#endif
353#ifndef HMVMX_USE_FUNCTION_TABLE
354DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
355# define HMVMX_EXIT_DECL static int
356#else
357# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
358#endif
359
360/** @name VM-exit handlers.
361 * @{
362 */
363static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
364static FNVMXEXITHANDLER hmR0VmxExitExtInt;
365static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
366static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
367static FNVMXEXITHANDLER hmR0VmxExitSipi;
368static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
369static FNVMXEXITHANDLER hmR0VmxExitSmi;
370static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
371static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
372static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
373static FNVMXEXITHANDLER hmR0VmxExitCpuid;
374static FNVMXEXITHANDLER hmR0VmxExitGetsec;
375static FNVMXEXITHANDLER hmR0VmxExitHlt;
376static FNVMXEXITHANDLER hmR0VmxExitInvd;
377static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
378static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
379static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
380static FNVMXEXITHANDLER hmR0VmxExitRsm;
381static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
382static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
383static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
384static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
385static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
386static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
387static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
388static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
389static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
390static FNVMXEXITHANDLER hmR0VmxExitMwait;
391static FNVMXEXITHANDLER hmR0VmxExitMtf;
392static FNVMXEXITHANDLER hmR0VmxExitMonitor;
393static FNVMXEXITHANDLER hmR0VmxExitPause;
394static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
395static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
396static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
397static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
398static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
404static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407/** @} */
408
409static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
417
418/*******************************************************************************
419* Global Variables *
420*******************************************************************************/
421#ifdef HMVMX_USE_FUNCTION_TABLE
422
423/**
424 * VMX_EXIT dispatch table.
425 */
426static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
427{
428 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
429 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
430 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
431 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
432 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
433 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
434 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
435 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
436 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
437 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
438 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
439 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
440 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
441 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
442 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
443 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
444 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
445 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
446 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
447 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
448 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
449 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
450 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
451 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
452 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
453 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
454 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
455 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
456 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
457 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
458 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
459 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
460 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
461 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
462 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
463 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
464 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
465 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
466 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
468 /* 40 UNDEFINED */ hmR0VmxExitPause,
469 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
470 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
471 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
472 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
473 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
474 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
475 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
476 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
477 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
478 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
479 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
480 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
481 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
482 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
483 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
484 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
485 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
486 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
487 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
488};
489#endif /* HMVMX_USE_FUNCTION_TABLE */
490
491#ifdef VBOX_STRICT
492static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
493{
494 /* 0 */ "(Not Used)",
495 /* 1 */ "VMCALL executed in VMX root operation.",
496 /* 2 */ "VMCLEAR with invalid physical address.",
497 /* 3 */ "VMCLEAR with VMXON pointer.",
498 /* 4 */ "VMLAUNCH with non-clear VMCS.",
499 /* 5 */ "VMRESUME with non-launched VMCS.",
500 /* 6 */ "VMRESUME after VMXOFF",
501 /* 7 */ "VM entry with invalid control fields.",
502 /* 8 */ "VM entry with invalid host state fields.",
503 /* 9 */ "VMPTRLD with invalid physical address.",
504 /* 10 */ "VMPTRLD with VMXON pointer.",
505 /* 11 */ "VMPTRLD with incorrect revision identifier.",
506 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
507 /* 13 */ "VMWRITE to read-only VMCS component.",
508 /* 14 */ "(Not Used)",
509 /* 15 */ "VMXON executed in VMX root operation.",
510 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
511 /* 17 */ "VM entry with non-launched executing VMCS.",
512 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
513 /* 19 */ "VMCALL with non-clear VMCS.",
514 /* 20 */ "VMCALL with invalid VM-exit control fields.",
515 /* 21 */ "(Not Used)",
516 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
517 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
518 /* 24 */ "VMCALL with invalid SMM-monitor features.",
519 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
520 /* 26 */ "VM entry with events blocked by MOV SS.",
521 /* 27 */ "(Not Used)",
522 /* 28 */ "Invalid operand to INVEPT/INVVPID."
523};
524#endif /* VBOX_STRICT */
525
526
527
528/**
529 * Updates the VM's last error record. If there was a VMX instruction error,
530 * reads the error data from the VMCS and updates VCPU's last error record as
531 * well.
532 *
533 * @param pVM Pointer to the VM.
534 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
535 * VERR_VMX_UNABLE_TO_START_VM or
536 * VERR_VMX_INVALID_VMCS_FIELD).
537 * @param rc The error code.
538 */
539static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
540{
541 AssertPtr(pVM);
542 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
543 || rc == VERR_VMX_UNABLE_TO_START_VM)
544 {
545 AssertPtrReturnVoid(pVCpu);
546 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
547 }
548 pVM->hm.s.lLastError = rc;
549}
550
551
552/**
553 * Reads the VM-entry interruption-information field from the VMCS into the VMX
554 * transient structure.
555 *
556 * @returns VBox status code.
557 * @param pVmxTransient Pointer to the VMX transient structure.
558 *
559 * @remarks No-long-jump zone!!!
560 */
561DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
562{
563 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
564 AssertRCReturn(rc, rc);
565 return VINF_SUCCESS;
566}
567
568
569/**
570 * Reads the VM-entry exception error code field from the VMCS into
571 * the VMX transient structure.
572 *
573 * @returns VBox status code.
574 * @param pVmxTransient Pointer to the VMX transient structure.
575 *
576 * @remarks No-long-jump zone!!!
577 */
578DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
579{
580 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
581 AssertRCReturn(rc, rc);
582 return VINF_SUCCESS;
583}
584
585
586/**
587 * Reads the VM-entry exception error code field from the VMCS into
588 * the VMX transient structure.
589 *
590 * @returns VBox status code.
591 * @param pVCpu Pointer to the VMCPU.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-exit interruption-information field from the VMCS into the VMX
606 * transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVCpu Pointer to the VMCPU.
610 * @param pVmxTransient Pointer to the VMX transient structure.
611 */
612DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
613{
614 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
615 {
616 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
617 AssertRCReturn(rc, rc);
618 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
619 }
620 return VINF_SUCCESS;
621}
622
623
624/**
625 * Reads the VM-exit interruption error code from the VMCS into the VMX
626 * transient structure.
627 *
628 * @returns VBox status code.
629 * @param pVCpu Pointer to the VMCPU.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 */
632DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
633{
634 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
635 {
636 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
637 AssertRCReturn(rc, rc);
638 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
639 }
640 return VINF_SUCCESS;
641}
642
643
644/**
645 * Reads the VM-exit instruction length field from the VMCS into the VMX
646 * transient structure.
647 *
648 * @returns VBox status code.
649 * @param pVCpu Pointer to the VMCPU.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
657 AssertRCReturn(rc, rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit instruction-information field from the VMCS into
666 * the VMX transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVCpu The cross context per CPU structure.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit qualification from the VMCS into the VMX transient structure.
686 *
687 * @returns VBox status code.
688 * @param pVCpu Pointer to the VMCPU.
689 * @param pVmxTransient Pointer to the VMX transient structure.
690 */
691DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
692{
693 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
694 {
695 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
696 AssertRCReturn(rc, rc);
697 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
698 }
699 return VINF_SUCCESS;
700}
701
702
703/**
704 * Reads the IDT-vectoring information field from the VMCS into the VMX
705 * transient structure.
706 *
707 * @returns VBox status code.
708 * @param pVmxTransient Pointer to the VMX transient structure.
709 *
710 * @remarks No-long-jump zone!!!
711 */
712DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
713{
714 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
715 {
716 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
717 AssertRCReturn(rc, rc);
718 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
719 }
720 return VINF_SUCCESS;
721}
722
723
724/**
725 * Reads the IDT-vectoring error code from the VMCS into the VMX
726 * transient structure.
727 *
728 * @returns VBox status code.
729 * @param pVmxTransient Pointer to the VMX transient structure.
730 */
731DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
732{
733 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
734 {
735 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
736 AssertRCReturn(rc, rc);
737 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
738 }
739 return VINF_SUCCESS;
740}
741
742
743/**
744 * Enters VMX root mode operation on the current CPU.
745 *
746 * @returns VBox status code.
747 * @param pVM Pointer to the VM (optional, can be NULL, after
748 * a resume).
749 * @param HCPhysCpuPage Physical address of the VMXON region.
750 * @param pvCpuPage Pointer to the VMXON region.
751 */
752static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
753{
754 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
755 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 if (pVM)
759 {
760 /* Write the VMCS revision dword to the VMXON region. */
761 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
762 }
763
764 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
765 RTCCUINTREG uEflags = ASMIntDisableFlags();
766
767 /* Enable the VMX bit in CR4 if necessary. */
768 RTCCUINTREG uCr4 = ASMGetCR4();
769 if (!(uCr4 & X86_CR4_VMXE))
770 ASMSetCR4(uCr4 | X86_CR4_VMXE);
771
772 /* Enter VMX root mode. */
773 int rc = VMXEnable(HCPhysCpuPage);
774 if (RT_FAILURE(rc))
775 ASMSetCR4(uCr4);
776
777 /* Restore interrupts. */
778 ASMSetFlags(uEflags);
779 return rc;
780}
781
782
783/**
784 * Exits VMX root mode operation on the current CPU.
785 *
786 * @returns VBox status code.
787 */
788static int hmR0VmxLeaveRootMode(void)
789{
790 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
791
792 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
793 RTCCUINTREG uEflags = ASMIntDisableFlags();
794
795 /* If we're for some reason not in VMX root mode, then don't leave it. */
796 RTCCUINTREG uHostCR4 = ASMGetCR4();
797
798 int rc;
799 if (uHostCR4 & X86_CR4_VMXE)
800 {
801 /* Exit VMX root mode and clear the VMX bit in CR4. */
802 VMXDisable();
803 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
804 rc = VINF_SUCCESS;
805 }
806 else
807 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
808
809 /* Restore interrupts. */
810 ASMSetFlags(uEflags);
811 return rc;
812}
813
814
815/**
816 * Allocates and maps one physically contiguous page. The allocated page is
817 * zero'd out. (Used by various VT-x structures).
818 *
819 * @returns IPRT status code.
820 * @param pMemObj Pointer to the ring-0 memory object.
821 * @param ppVirt Where to store the virtual address of the
822 * allocation.
823 * @param pPhys Where to store the physical address of the
824 * allocation.
825 */
826DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
827{
828 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
829 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
830 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
831
832 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
833 if (RT_FAILURE(rc))
834 return rc;
835 *ppVirt = RTR0MemObjAddress(*pMemObj);
836 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
837 ASMMemZero32(*ppVirt, PAGE_SIZE);
838 return VINF_SUCCESS;
839}
840
841
842/**
843 * Frees and unmaps an allocated physical page.
844 *
845 * @param pMemObj Pointer to the ring-0 memory object.
846 * @param ppVirt Where to re-initialize the virtual address of
847 * allocation as 0.
848 * @param pHCPhys Where to re-initialize the physical address of the
849 * allocation as 0.
850 */
851DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
852{
853 AssertPtr(pMemObj);
854 AssertPtr(ppVirt);
855 AssertPtr(pHCPhys);
856 if (*pMemObj != NIL_RTR0MEMOBJ)
857 {
858 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
859 AssertRC(rc);
860 *pMemObj = NIL_RTR0MEMOBJ;
861 *ppVirt = 0;
862 *pHCPhys = 0;
863 }
864}
865
866
867/**
868 * Worker function to free VT-x related structures.
869 *
870 * @returns IPRT status code.
871 * @param pVM Pointer to the VM.
872 */
873static void hmR0VmxStructsFree(PVM pVM)
874{
875 for (VMCPUID i = 0; i < pVM->cCpus; i++)
876 {
877 PVMCPU pVCpu = &pVM->aCpus[i];
878 AssertPtr(pVCpu);
879
880#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
881 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
883#endif
884
885 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
887
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
890 }
891
892 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
893#ifdef VBOX_WITH_CRASHDUMP_MAGIC
894 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
895#endif
896}
897
898
899/**
900 * Worker function to allocate VT-x related VM structures.
901 *
902 * @returns IPRT status code.
903 * @param pVM Pointer to the VM.
904 */
905static int hmR0VmxStructsAlloc(PVM pVM)
906{
907 /*
908 * Initialize members up-front so we can cleanup properly on allocation failure.
909 */
910#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
911 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
912 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
913 pVM->hm.s.vmx.HCPhys##a_Name = 0;
914
915#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
919
920#ifdef VBOX_WITH_CRASHDUMP_MAGIC
921 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
922#endif
923 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
924
925 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
926 for (VMCPUID i = 0; i < pVM->cCpus; i++)
927 {
928 PVMCPU pVCpu = &pVM->aCpus[i];
929 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
931 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
932#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935#endif
936 }
937#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
938#undef VMXLOCAL_INIT_VM_MEMOBJ
939
940 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
941 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
942 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
943 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
944
945 /*
946 * Allocate all the VT-x structures.
947 */
948 int rc = VINF_SUCCESS;
949#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951 if (RT_FAILURE(rc))
952 goto cleanup;
953 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
954 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
955#endif
956
957 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
958 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
959 {
960 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
961 &pVM->hm.s.vmx.HCPhysApicAccess);
962 if (RT_FAILURE(rc))
963 goto cleanup;
964 }
965
966 /*
967 * Initialize per-VCPU VT-x structures.
968 */
969 for (VMCPUID i = 0; i < pVM->cCpus; i++)
970 {
971 PVMCPU pVCpu = &pVM->aCpus[i];
972 AssertPtr(pVCpu);
973
974 /* Allocate the VM control structure (VMCS). */
975 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978
979 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
980 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
981 {
982 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
983 &pVCpu->hm.s.vmx.HCPhysVirtApic);
984 if (RT_FAILURE(rc))
985 goto cleanup;
986 }
987
988 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
990 {
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
992 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
996 }
997
998#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
999 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003
1004 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1005 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008#endif
1009 }
1010
1011 return VINF_SUCCESS;
1012
1013cleanup:
1014 hmR0VmxStructsFree(pVM);
1015 return rc;
1016}
1017
1018
1019/**
1020 * Does global VT-x initialization (called during module initialization).
1021 *
1022 * @returns VBox status code.
1023 */
1024VMMR0DECL(int) VMXR0GlobalInit(void)
1025{
1026#ifdef HMVMX_USE_FUNCTION_TABLE
1027 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1028# ifdef VBOX_STRICT
1029 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1030 Assert(g_apfnVMExitHandlers[i]);
1031# endif
1032#endif
1033 return VINF_SUCCESS;
1034}
1035
1036
1037/**
1038 * Does global VT-x termination (called during module termination).
1039 */
1040VMMR0DECL(void) VMXR0GlobalTerm()
1041{
1042 /* Nothing to do currently. */
1043}
1044
1045
1046/**
1047 * Sets up and activates VT-x on the current CPU.
1048 *
1049 * @returns VBox status code.
1050 * @param pCpu Pointer to the global CPU info struct.
1051 * @param pVM Pointer to the VM (can be NULL after a host resume
1052 * operation).
1053 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1054 * fEnabledByHost is true).
1055 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1056 * @a fEnabledByHost is true).
1057 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1058 * enable VT-x on the host.
1059 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1060 */
1061VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1062 void *pvMsrs)
1063{
1064 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1065 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067
1068 /* Enable VT-x if it's not already enabled by the host. */
1069 if (!fEnabledByHost)
1070 {
1071 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1072 if (RT_FAILURE(rc))
1073 return rc;
1074 }
1075
1076 /*
1077 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1078 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1079 */
1080 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1081 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1082 {
1083 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1084 pCpu->fFlushAsidBeforeUse = false;
1085 }
1086 else
1087 pCpu->fFlushAsidBeforeUse = true;
1088
1089 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1090 ++pCpu->cTlbFlushes;
1091
1092 return VINF_SUCCESS;
1093}
1094
1095
1096/**
1097 * Deactivates VT-x on the current CPU.
1098 *
1099 * @returns VBox status code.
1100 * @param pCpu Pointer to the global CPU info struct.
1101 * @param pvCpuPage Pointer to the VMXON region.
1102 * @param HCPhysCpuPage Physical address of the VMXON region.
1103 *
1104 * @remarks This function should never be called when SUPR0EnableVTx() or
1105 * similar was used to enable VT-x on the host.
1106 */
1107VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1108{
1109 NOREF(pCpu);
1110 NOREF(pvCpuPage);
1111 NOREF(HCPhysCpuPage);
1112
1113 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1114 return hmR0VmxLeaveRootMode();
1115}
1116
1117
1118/**
1119 * Sets the permission bits for the specified MSR in the MSR bitmap.
1120 *
1121 * @param pVCpu Pointer to the VMCPU.
1122 * @param uMSR The MSR value.
1123 * @param enmRead Whether reading this MSR causes a VM-exit.
1124 * @param enmWrite Whether writing this MSR causes a VM-exit.
1125 */
1126static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1127{
1128 int32_t iBit;
1129 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1130
1131 /*
1132 * Layout:
1133 * 0x000 - 0x3ff - Low MSR read bits
1134 * 0x400 - 0x7ff - High MSR read bits
1135 * 0x800 - 0xbff - Low MSR write bits
1136 * 0xc00 - 0xfff - High MSR write bits
1137 */
1138 if (uMsr <= 0x00001FFF)
1139 iBit = uMsr;
1140 else if ( uMsr >= 0xC0000000
1141 && uMsr <= 0xC0001FFF)
1142 {
1143 iBit = (uMsr - 0xC0000000);
1144 pbMsrBitmap += 0x400;
1145 }
1146 else
1147 {
1148 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1149 return;
1150 }
1151
1152 Assert(iBit <= 0x1fff);
1153 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154 ASMBitSet(pbMsrBitmap, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap, iBit);
1157
1158 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160 else
1161 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162}
1163
1164
1165#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1166/**
1167 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1168 * area.
1169 *
1170 * @returns VBox status code.
1171 * @param pVCpu Pointer to the VMCPU.
1172 * @param cMsrs The number of MSRs.
1173 */
1174DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1175{
1176 /* Update the VCPU's copy of the guest MSR count. */
1177 pVCpu->hm.s.vmx.cGuestMsrs = cMsrs;
1178
1179 /* Update number of guest MSRs to load/store across the world-switch. */
1180 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1181 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1182
1183 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1184 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1185 return VINF_SUCCESS;
1186}
1187
1188
1189/**
1190 * Adds a guest/host MSR pair to be swapped during the world-switch as
1191 * part of the auto-load/store MSR area in the VMCS.
1192 *
1193 * @returns VBox status code.
1194 * @param pVCpu Pointer to the VMCPU.
1195 * @param uMsr The MSR.
1196 * @param uGuestMsr Value of the guest MSR.
1197 * @param uHostMsr Value of the host MSR.
1198 */
1199static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, uint64_t uHostMsrValue)
1200{
1201 AssertMsg(HMVMX_MAX_SWAP_MSR_COUNT < MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc),
1202 ("MSR swap count exceeded. Cpu reports %#RX32, our limit %#RX32\n",
1203 MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc), HMVMX_MAX_SWAP_MSR_COUNT));
1204
1205 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1206 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1207 uint32_t i;
1208 for (i = 0; i < cGuestMsrs; i++)
1209 {
1210 if (pGuestMsr->u32Msr == uMsr)
1211 break;
1212 pGuestMsr++;
1213 }
1214
1215 AssertReturn(i < HMVMX_MAX_SWAP_MSR_COUNT, VERR_HM_MSR_SWAP_COUNT_EXCEEDED);
1216 if (i == cGuestMsrs)
1217 {
1218 ++cGuestMsrs;
1219 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1220 if (RT_UNLIKELY(cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc)))
1221 {
1222 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
1223 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1224 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1225 }
1226
1227 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1228 AssertRCReturn(rc, rc);
1229 }
1230
1231 /* Update the MSR values in the auto-load/store MSR area. */
1232 pGuestMsr->u32Msr = uMsr;
1233 pGuestMsr->u64Value = uGuestMsrValue;
1234
1235 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1236 pHostMsr += i;
1237 pHostMsr->u32Msr = uMsr;
1238 pHostMsr->u64Value = uHostMsrValue;
1239
1240 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1241 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
1242 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1243 return VINF_SUCCESS;
1244}
1245
1246
1247/**
1248 * Removes a guest/shost MSR pair to be swapped during the world-switch from the
1249 * auto-load/store MSR area in the VMCS.
1250 *
1251 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1252 * area.
1253 *
1254 * @returns VBox status code.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 */
1258static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cGuestMsrs; i++)
1264 {
1265 /* Find the MSR. */
1266 if (pGuestMsr->u32Msr == uMsr)
1267 {
1268 /* If it's the last MSR, simply reduce the count. */
1269 if (i == cGuestMsrs - 1)
1270 {
1271 --cGuestMsrs;
1272 break;
1273 }
1274
1275 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1276 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1277 pLastGuestMsr += cGuestMsrs;
1278 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1279 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1280
1281 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1282 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1283 pLastHostMsr += cGuestMsrs;
1284 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1285 pHostMsr->u64Value = pLastHostMsr->u64Value;
1286 --cGuestMsrs;
1287 break;
1288 }
1289 pGuestMsr++;
1290 }
1291
1292 /* Update the VMCS if the count changed (meaning the MSR was found). */
1293 if (cGuestMsrs != pVCpu->hm.s.vmx.cGuestMsrs)
1294 {
1295 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1296 AssertRCReturn(rc, rc);
1297 }
1298
1299 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1300 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1301 return VINF_SUCCESS;
1302}
1303
1304
1305/**
1306 * Updates the value of a host MSR in the auto-load/store area in the VMCS.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxUpdateAutoLoadStoreHostMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1316
1317 for (uint32_t i = 0; i < cMsrs; i++)
1318 {
1319 if (pHostMsr->u32Msr == uMsr)
1320 {
1321 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1322 return VINF_SUCCESS;
1323 }
1324 }
1325
1326 return VERR_NOT_FOUND;
1327}
1328
1329#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1330
1331
1332/**
1333 * Flushes the TLB using EPT.
1334 *
1335 * @returns VBox status code.
1336 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1337 * enmFlush).
1338 * @param enmFlush Type of flush.
1339 *
1340 * @remarks Caller is responsible for making sure this function is called only
1341 * when NestedPaging is supported and providing @a enmFlush that is
1342 * supported by the CPU.
1343 * @remarks Can be called with interrupts disabled.
1344 */
1345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1346{
1347 uint64_t au64Descriptor[2];
1348 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1349 au64Descriptor[0] = 0;
1350 else
1351 {
1352 Assert(pVCpu);
1353 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1354 }
1355 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1356
1357 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1358 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1359 rc));
1360 if ( RT_SUCCESS(rc)
1361 && pVCpu)
1362 {
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1364 }
1365}
1366
1367
1368/**
1369 * Flushes the TLB using VPID.
1370 *
1371 * @returns VBox status code.
1372 * @param pVM Pointer to the VM.
1373 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1374 * enmFlush).
1375 * @param enmFlush Type of flush.
1376 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1377 * on @a enmFlush).
1378 *
1379 * @remarks Can be called with interrupts disabled.
1380 */
1381static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1382{
1383 AssertPtr(pVM);
1384 Assert(pVM->hm.s.vmx.fVpid);
1385
1386 uint64_t au64Descriptor[2];
1387 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1388 {
1389 au64Descriptor[0] = 0;
1390 au64Descriptor[1] = 0;
1391 }
1392 else
1393 {
1394 AssertPtr(pVCpu);
1395 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1396 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1397 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1398 au64Descriptor[1] = GCPtr;
1399 }
1400
1401 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1402 AssertMsg(rc == VINF_SUCCESS,
1403 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1404 if ( RT_SUCCESS(rc)
1405 && pVCpu)
1406 {
1407 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1408 }
1409}
1410
1411
1412/**
1413 * Invalidates a guest page by guest virtual address. Only relevant for
1414 * EPT/VPID, otherwise there is nothing really to invalidate.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param GCVirt Guest virtual address of the page to invalidate.
1420 */
1421VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1422{
1423 AssertPtr(pVM);
1424 AssertPtr(pVCpu);
1425 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1426
1427 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1428 if (!fFlushPending)
1429 {
1430 /*
1431 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1432 * See @bugref{6043} and @bugref{6177}.
1433 *
1434 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1435 * function maybe called in a loop with individual addresses.
1436 */
1437 if (pVM->hm.s.vmx.fVpid)
1438 {
1439 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1440 {
1441 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1442 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1443 }
1444 else
1445 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1446 }
1447 else if (pVM->hm.s.fNestedPaging)
1448 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1449 }
1450
1451 return VINF_SUCCESS;
1452}
1453
1454
1455/**
1456 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1457 * otherwise there is nothing really to invalidate.
1458 *
1459 * @returns VBox status code.
1460 * @param pVM Pointer to the VM.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param GCPhys Guest physical address of the page to invalidate.
1463 */
1464VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1465{
1466 LogFlowFunc(("%RGp\n", GCPhys));
1467
1468 /*
1469 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1470 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1471 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1472 */
1473 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1474 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1475 return VINF_SUCCESS;
1476}
1477
1478
1479/**
1480 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1481 * case where neither EPT nor VPID is supported by the CPU.
1482 *
1483 * @param pVM Pointer to the VM.
1484 * @param pVCpu Pointer to the VMCPU.
1485 * @param pCpu Pointer to the global HM struct.
1486 *
1487 * @remarks Called with interrupts disabled.
1488 */
1489static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1490{
1491 AssertPtr(pVCpu);
1492 AssertPtr(pCpu);
1493 NOREF(pVM);
1494
1495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1496
1497 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1498#if 0
1499 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1500 pVCpu->hm.s.TlbShootdown.cPages = 0;
1501#endif
1502
1503 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1504 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1505 pVCpu->hm.s.fForceTLBFlush = false;
1506 return;
1507}
1508
1509
1510/**
1511 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1512 *
1513 * @param pVM Pointer to the VM.
1514 * @param pVCpu Pointer to the VMCPU.
1515 * @param pCpu Pointer to the global HM CPU struct.
1516 * @remarks All references to "ASID" in this function pertains to "VPID" in
1517 * Intel's nomenclature. The reason is, to avoid confusion in compare
1518 * statements since the host-CPU copies are named "ASID".
1519 *
1520 * @remarks Called with interrupts disabled.
1521 */
1522static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1523{
1524#ifdef VBOX_WITH_STATISTICS
1525 bool fTlbFlushed = false;
1526# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1527# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1528 if (!fTlbFlushed) \
1529 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1530 } while (0)
1531#else
1532# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1533# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1534#endif
1535
1536 AssertPtr(pVM);
1537 AssertPtr(pCpu);
1538 AssertPtr(pVCpu);
1539 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1540 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1541 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1542
1543
1544 /*
1545 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1546 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1547 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1548 */
1549 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1550 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1551 {
1552 ++pCpu->uCurrentAsid;
1553 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1554 {
1555 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1556 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1557 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1558 }
1559
1560 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1561 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1562 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1563
1564 /*
1565 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1566 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1567 */
1568 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1569 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1570 HMVMX_SET_TAGGED_TLB_FLUSHED();
1571 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1572 }
1573
1574 /* Check for explicit TLB shootdowns. */
1575 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1576 {
1577 /*
1578 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1579 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1580 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1581 * but not guest-physical mappings.
1582 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1583 */
1584 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1585 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1586 HMVMX_SET_TAGGED_TLB_FLUSHED();
1587 }
1588
1589 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1590 * where it is commented out. Support individual entry flushing
1591 * someday. */
1592#if 0
1593 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1594 {
1595 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1596
1597 /*
1598 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1599 * as supported by the CPU.
1600 */
1601 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1602 {
1603 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1604 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1605 }
1606 else
1607 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1608
1609 HMVMX_SET_TAGGED_TLB_FLUSHED();
1610 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1611 pVCpu->hm.s.TlbShootdown.cPages = 0;
1612 }
1613#endif
1614
1615 pVCpu->hm.s.fForceTLBFlush = false;
1616
1617 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1618
1619 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1620 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1621 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1622 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1623 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1624 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1625 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1626 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1627
1628 /* Update VMCS with the VPID. */
1629 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1630 AssertRC(rc);
1631
1632#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1633}
1634
1635
1636/**
1637 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1638 *
1639 * @returns VBox status code.
1640 * @param pVM Pointer to the VM.
1641 * @param pVCpu Pointer to the VMCPU.
1642 * @param pCpu Pointer to the global HM CPU struct.
1643 *
1644 * @remarks Called with interrupts disabled.
1645 */
1646static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1647{
1648 AssertPtr(pVM);
1649 AssertPtr(pVCpu);
1650 AssertPtr(pCpu);
1651 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1652 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1653
1654 /*
1655 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1656 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1657 */
1658 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1659 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1660 {
1661 pVCpu->hm.s.fForceTLBFlush = true;
1662 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1663 }
1664
1665 /* Check for explicit TLB shootdown flushes. */
1666 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1667 {
1668 pVCpu->hm.s.fForceTLBFlush = true;
1669 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1670 }
1671
1672 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1673 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1674
1675 if (pVCpu->hm.s.fForceTLBFlush)
1676 {
1677 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1678 pVCpu->hm.s.fForceTLBFlush = false;
1679 }
1680 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1681 * where it is commented out. Support individual entry flushing
1682 * someday. */
1683#if 0
1684 else
1685 {
1686 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1687 {
1688 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1689 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1690 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1691 }
1692 else
1693 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1694
1695 pVCpu->hm.s.TlbShootdown.cPages = 0;
1696 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1697 }
1698#endif
1699}
1700
1701
1702/**
1703 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1704 *
1705 * @returns VBox status code.
1706 * @param pVM Pointer to the VM.
1707 * @param pVCpu Pointer to the VMCPU.
1708 * @param pCpu Pointer to the global HM CPU struct.
1709 *
1710 * @remarks Called with interrupts disabled.
1711 */
1712static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1713{
1714 AssertPtr(pVM);
1715 AssertPtr(pVCpu);
1716 AssertPtr(pCpu);
1717 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1718 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1719
1720 /*
1721 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1722 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1723 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1724 */
1725 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1726 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1727 {
1728 pVCpu->hm.s.fForceTLBFlush = true;
1729 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1730 }
1731
1732 /* Check for explicit TLB shootdown flushes. */
1733 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1734 {
1735 /*
1736 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1737 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1738 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1739 */
1740 pVCpu->hm.s.fForceTLBFlush = true;
1741 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1742 }
1743
1744 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1745 if (pVCpu->hm.s.fForceTLBFlush)
1746 {
1747 ++pCpu->uCurrentAsid;
1748 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1749 {
1750 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1751 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1752 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1753 }
1754
1755 pVCpu->hm.s.fForceTLBFlush = false;
1756 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1757 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1758 if (pCpu->fFlushAsidBeforeUse)
1759 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1760 }
1761 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1762 * where it is commented out. Support individual entry flushing
1763 * someday. */
1764#if 0
1765 else
1766 {
1767 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1768 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1769 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1770 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1771
1772 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1773 {
1774 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1776 {
1777 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1778 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1779 }
1780 else
1781 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1782
1783 pVCpu->hm.s.TlbShootdown.cPages = 0;
1784 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1785 }
1786 else
1787 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1788 }
1789#endif
1790
1791 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1792 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1793 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1794 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1795 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1796 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1797
1798 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1799 AssertRC(rc);
1800}
1801
1802
1803/**
1804 * Flushes the guest TLB entry based on CPU capabilities.
1805 *
1806 * @param pVCpu Pointer to the VMCPU.
1807 * @param pCpu Pointer to the global HM CPU struct.
1808 */
1809DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1810{
1811#ifdef HMVMX_ALWAYS_FLUSH_TLB
1812 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1813#endif
1814 PVM pVM = pVCpu->CTX_SUFF(pVM);
1815 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1816 {
1817 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1818 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1819 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1820 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1821 default:
1822 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1823 break;
1824 }
1825
1826 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
1827 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
1828
1829 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
1830}
1831
1832
1833/**
1834 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1835 * TLB entries from the host TLB before VM-entry.
1836 *
1837 * @returns VBox status code.
1838 * @param pVM Pointer to the VM.
1839 */
1840static int hmR0VmxSetupTaggedTlb(PVM pVM)
1841{
1842 /*
1843 * Determine optimal flush type for Nested Paging.
1844 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1845 * guest execution (see hmR3InitFinalizeR0()).
1846 */
1847 if (pVM->hm.s.fNestedPaging)
1848 {
1849 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1850 {
1851 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1852 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1853 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1854 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1855 else
1856 {
1857 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1858 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1859 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1860 }
1861
1862 /* Make sure the write-back cacheable memory type for EPT is supported. */
1863 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1864 {
1865 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1866 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1867 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1868 }
1869 }
1870 else
1871 {
1872 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1873 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1874 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1875 }
1876 }
1877
1878 /*
1879 * Determine optimal flush type for VPID.
1880 */
1881 if (pVM->hm.s.vmx.fVpid)
1882 {
1883 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1884 {
1885 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1886 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1887 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1888 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1889 else
1890 {
1891 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1892 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1893 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1894 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1895 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1896 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1897 pVM->hm.s.vmx.fVpid = false;
1898 }
1899 }
1900 else
1901 {
1902 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1903 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1904 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1905 pVM->hm.s.vmx.fVpid = false;
1906 }
1907 }
1908
1909 /*
1910 * Setup the handler for flushing tagged-TLBs.
1911 */
1912 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1913 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1914 else if (pVM->hm.s.fNestedPaging)
1915 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1916 else if (pVM->hm.s.vmx.fVpid)
1917 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1918 else
1919 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1920 return VINF_SUCCESS;
1921}
1922
1923
1924/**
1925 * Sets up pin-based VM-execution controls in the VMCS.
1926 *
1927 * @returns VBox status code.
1928 * @param pVM Pointer to the VM.
1929 * @param pVCpu Pointer to the VMCPU.
1930 */
1931static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1932{
1933 AssertPtr(pVM);
1934 AssertPtr(pVCpu);
1935
1936 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1937 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1938
1939 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1940 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1941 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1942
1943 /* Enable the VMX preemption timer. */
1944 if (pVM->hm.s.vmx.fUsePreemptTimer)
1945 {
1946 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1947 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1948 }
1949
1950 if ((val & zap) != val)
1951 {
1952 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1953 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1954 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1955 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1956 }
1957
1958 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1959 AssertRCReturn(rc, rc);
1960
1961 /* Update VCPU with the currently set pin-based VM-execution controls. */
1962 pVCpu->hm.s.vmx.u32PinCtls = val;
1963 return rc;
1964}
1965
1966
1967/**
1968 * Sets up processor-based VM-execution controls in the VMCS.
1969 *
1970 * @returns VBox status code.
1971 * @param pVM Pointer to the VM.
1972 * @param pVMCPU Pointer to the VMCPU.
1973 */
1974static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1975{
1976 AssertPtr(pVM);
1977 AssertPtr(pVCpu);
1978
1979 int rc = VERR_INTERNAL_ERROR_5;
1980 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1981 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1982
1983 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1984 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1985 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1986 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1987 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1988 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1989 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1990
1991 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1992 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1993 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1994 {
1995 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1996 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1997 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1998 }
1999
2000 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2001 if (!pVM->hm.s.fNestedPaging)
2002 {
2003 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2004 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2005 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2006 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2007 }
2008
2009 /* Use TPR shadowing if supported by the CPU. */
2010 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2011 {
2012 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2013 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2014 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2015 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2016 AssertRCReturn(rc, rc);
2017
2018 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2019 /* CR8 writes causes a VM-exit based on TPR threshold. */
2020 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2021 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2022 }
2023 else
2024 {
2025 /*
2026 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2027 * Set this control only for 64-bit guests.
2028 */
2029 if (pVM->hm.s.fAllow64BitGuests)
2030 {
2031 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2032 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2033 }
2034 }
2035
2036 /* Use MSR-bitmaps if supported by the CPU. */
2037 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2038 {
2039 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2040
2041 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2042 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2043 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2044 AssertRCReturn(rc, rc);
2045
2046 /*
2047 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2048 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
2049 */
2050 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2051 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2052 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2053 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2054 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2055 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2056 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2057 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2058 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2059 }
2060
2061 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2062 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2063 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2064
2065 if ((val & zap) != val)
2066 {
2067 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2068 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2069 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2070 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2071 }
2072
2073 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2074 AssertRCReturn(rc, rc);
2075
2076 /* Update VCPU with the currently set processor-based VM-execution controls. */
2077 pVCpu->hm.s.vmx.u32ProcCtls = val;
2078
2079 /*
2080 * Secondary processor-based VM-execution controls.
2081 */
2082 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2083 {
2084 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2085 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2086
2087 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2088 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2089
2090 if (pVM->hm.s.fNestedPaging)
2091 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2092 else
2093 {
2094 /*
2095 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2096 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2097 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2098 */
2099 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2100 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2101 }
2102
2103 if (pVM->hm.s.vmx.fVpid)
2104 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2105
2106 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2107 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2108
2109 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2110 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2111 * done dynamically. */
2112 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2113 {
2114 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2115 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2116 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2117 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2118 AssertRCReturn(rc, rc);
2119 }
2120
2121 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2122 {
2123 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2124 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2125 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2126 }
2127
2128 if ((val & zap) != val)
2129 {
2130 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2131 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2132 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2133 }
2134
2135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2136 AssertRCReturn(rc, rc);
2137
2138 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2139 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2140 }
2141 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2142 {
2143 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2144 "available\n"));
2145 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2146 }
2147
2148 return VINF_SUCCESS;
2149}
2150
2151
2152/**
2153 * Sets up miscellaneous (everything other than Pin & Processor-based
2154 * VM-execution) control fields in the VMCS.
2155 *
2156 * @returns VBox status code.
2157 * @param pVM Pointer to the VM.
2158 * @param pVCpu Pointer to the VMCPU.
2159 */
2160static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2161{
2162 AssertPtr(pVM);
2163 AssertPtr(pVCpu);
2164
2165 int rc = VERR_GENERAL_FAILURE;
2166
2167 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2168#if 0
2169 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2170 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2171 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2172
2173 /*
2174 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2175 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2176 * We thus use the exception bitmap to control it rather than use both.
2177 */
2178 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2179 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2180
2181 /** @todo Explore possibility of using IO-bitmaps. */
2182 /* All IO & IOIO instructions cause VM-exits. */
2183 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2184 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2185
2186 /* Initialize the MSR-bitmap area. */
2187 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2188 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2189 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2190#endif
2191
2192#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2193 /* Setup MSR autoloading/storing. */
2194 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2195 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2196 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2197 AssertRCReturn(rc, rc);
2198 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2199 AssertRCReturn(rc, rc);
2200
2201 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2202 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2203 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2204 AssertRCReturn(rc, rc);
2205#endif
2206
2207 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2208 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2209 AssertRCReturn(rc, rc);
2210
2211 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2212#if 0
2213 /* Setup debug controls */
2214 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2215 AssertRCReturn(rc, rc);
2216 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2217 AssertRCReturn(rc, rc);
2218#endif
2219
2220 return rc;
2221}
2222
2223
2224/**
2225 * Sets up the initial exception bitmap in the VMCS based on static conditions
2226 * (i.e. conditions that cannot ever change after starting the VM).
2227 *
2228 * @returns VBox status code.
2229 * @param pVM Pointer to the VM.
2230 * @param pVCpu Pointer to the VMCPU.
2231 */
2232static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2233{
2234 AssertPtr(pVM);
2235 AssertPtr(pVCpu);
2236
2237 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2238
2239 uint32_t u32XcptBitmap = 0;
2240
2241 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2242 if (!pVM->hm.s.fNestedPaging)
2243 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2244
2245 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2246 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2247 AssertRCReturn(rc, rc);
2248 return rc;
2249}
2250
2251
2252/**
2253 * Sets up the initial guest-state mask. The guest-state mask is consulted
2254 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2255 * for the nested virtualization case (as it would cause a VM-exit).
2256 *
2257 * @param pVCpu Pointer to the VMCPU.
2258 */
2259static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2260{
2261 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2262 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Does per-VM VT-x initialization.
2269 *
2270 * @returns VBox status code.
2271 * @param pVM Pointer to the VM.
2272 */
2273VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2274{
2275 LogFlowFunc(("pVM=%p\n", pVM));
2276
2277 int rc = hmR0VmxStructsAlloc(pVM);
2278 if (RT_FAILURE(rc))
2279 {
2280 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2281 return rc;
2282 }
2283
2284 return VINF_SUCCESS;
2285}
2286
2287
2288/**
2289 * Does per-VM VT-x termination.
2290 *
2291 * @returns VBox status code.
2292 * @param pVM Pointer to the VM.
2293 */
2294VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2295{
2296 LogFlowFunc(("pVM=%p\n", pVM));
2297
2298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2299 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2300 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2301#endif
2302 hmR0VmxStructsFree(pVM);
2303 return VINF_SUCCESS;
2304}
2305
2306
2307/**
2308 * Sets up the VM for execution under VT-x.
2309 * This function is only called once per-VM during initialization.
2310 *
2311 * @returns VBox status code.
2312 * @param pVM Pointer to the VM.
2313 */
2314VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2315{
2316 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2317 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2318
2319 LogFlowFunc(("pVM=%p\n", pVM));
2320
2321 /*
2322 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2323 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2324 */
2325 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2326 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2327 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2328 || !pVM->hm.s.vmx.pRealModeTSS))
2329 {
2330 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2331 return VERR_INTERNAL_ERROR;
2332 }
2333
2334#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2335 /*
2336 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2337 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2338 */
2339 if ( pVM->hm.s.fAllow64BitGuests
2340 && !HMVMX_IS_64BIT_HOST_MODE())
2341 {
2342 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2343 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2344 }
2345#endif
2346
2347 /* Initialize these always, see hmR3InitFinalizeR0().*/
2348 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2349 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2350
2351 /* Setup the tagged-TLB flush handlers. */
2352 int rc = hmR0VmxSetupTaggedTlb(pVM);
2353 if (RT_FAILURE(rc))
2354 {
2355 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2356 return rc;
2357 }
2358
2359 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2360 {
2361 PVMCPU pVCpu = &pVM->aCpus[i];
2362 AssertPtr(pVCpu);
2363 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2364
2365 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2366 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2367
2368 /* Set revision dword at the beginning of the VMCS structure. */
2369 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2370
2371 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2372 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2373 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2374 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2375
2376 /* Load this VMCS as the current VMCS. */
2377 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2378 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2379 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2380
2381 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2382 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2383 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2384
2385 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2386 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2387 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2388
2389 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2390 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2391 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2392
2393 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2394 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2395 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2396
2397 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2398 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2399 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2400
2401#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2402 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2403 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2404 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2405#endif
2406
2407 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2408 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2409 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2410 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2411
2412 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2413
2414 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2415 }
2416
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2423 * the VMCS.
2424 *
2425 * @returns VBox status code.
2426 * @param pVM Pointer to the VM.
2427 * @param pVCpu Pointer to the VMCPU.
2428 */
2429DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2430{
2431 RTCCUINTREG uReg = ASMGetCR0();
2432 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2433 AssertRCReturn(rc, rc);
2434
2435#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2436 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2437 if (HMVMX_IS_64BIT_HOST_MODE())
2438 {
2439 uint64_t uRegCR3 = HMR0Get64bitCR3();
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2441 }
2442 else
2443#endif
2444 {
2445 uReg = ASMGetCR3();
2446 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2447 }
2448 AssertRCReturn(rc, rc);
2449
2450 uReg = ASMGetCR4();
2451 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2452 AssertRCReturn(rc, rc);
2453 return rc;
2454}
2455
2456
2457#if HC_ARCH_BITS == 64
2458/**
2459 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2460 * requirements. See hmR0VmxSaveHostSegmentRegs().
2461 */
2462# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2463 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2464 { \
2465 bool fValidSelector = true; \
2466 if ((selValue) & X86_SEL_LDT) \
2467 { \
2468 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2469 fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2470 } \
2471 if (fValidSelector) \
2472 { \
2473 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2474 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2475 } \
2476 (selValue) = 0; \
2477 }
2478#endif
2479
2480
2481/**
2482 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2483 * the host-state area in the VMCS.
2484 *
2485 * @returns VBox status code.
2486 * @param pVM Pointer to the VM.
2487 * @param pVCpu Pointer to the VMCPU.
2488 */
2489DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2490{
2491 int rc = VERR_INTERNAL_ERROR_5;
2492
2493 /*
2494 * Host DS, ES, FS and GS segment registers.
2495 */
2496#if HC_ARCH_BITS == 64
2497 RTSEL uSelDS = ASMGetDS();
2498 RTSEL uSelES = ASMGetES();
2499 RTSEL uSelFS = ASMGetFS();
2500 RTSEL uSelGS = ASMGetGS();
2501#else
2502 RTSEL uSelDS = 0;
2503 RTSEL uSelES = 0;
2504 RTSEL uSelFS = 0;
2505 RTSEL uSelGS = 0;
2506#endif
2507
2508 /* Recalculate which host-state bits need to be manually restored. */
2509 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2510
2511 /*
2512 * Host CS and SS segment registers.
2513 */
2514#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2515 RTSEL uSelCS;
2516 RTSEL uSelSS;
2517 if (HMVMX_IS_64BIT_HOST_MODE())
2518 {
2519 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2520 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2521 }
2522 else
2523 {
2524 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2525 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2526 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2527 }
2528#else
2529 RTSEL uSelCS = ASMGetCS();
2530 RTSEL uSelSS = ASMGetSS();
2531#endif
2532
2533 /*
2534 * Host TR segment register.
2535 */
2536 RTSEL uSelTR = ASMGetTR();
2537
2538#if HC_ARCH_BITS == 64
2539 /*
2540 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2541 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2542 */
2543 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2544 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2545 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2546 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2547# undef VMXLOCAL_ADJUST_HOST_SEG
2548#endif
2549
2550 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2551 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2552 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2553 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2554 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2555 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2556 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2557 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2558 Assert(uSelCS);
2559 Assert(uSelTR);
2560
2561 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2562#if 0
2563 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2564 Assert(uSelSS != 0);
2565#endif
2566
2567 /* Write these host selector fields into the host-state area in the VMCS. */
2568 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2570#if HC_ARCH_BITS == 64
2571 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2575#endif
2576 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2577
2578 /*
2579 * Host GDTR and IDTR.
2580 */
2581 RTGDTR Gdtr;
2582 RT_ZERO(Gdtr);
2583#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2584 if (HMVMX_IS_64BIT_HOST_MODE())
2585 {
2586 X86XDTR64 Gdtr64;
2587 X86XDTR64 Idtr64;
2588 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2589 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2591
2592 Gdtr.cbGdt = Gdtr64.cb;
2593 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2594 }
2595 else
2596#endif
2597 {
2598 RTIDTR Idtr;
2599 ASMGetGDTR(&Gdtr);
2600 ASMGetIDTR(&Idtr);
2601 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2602 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2603
2604#if HC_ARCH_BITS == 64
2605 /*
2606 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2607 * maximum limit (0xffff) on every VM-exit.
2608 */
2609 if (Gdtr.cbGdt != 0xffff)
2610 {
2611 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2612 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2613 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2614 }
2615
2616 /*
2617 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2618 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2619 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2620 */
2621 if (Idtr.cbIdt < 0x0fff)
2622 {
2623 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2624 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2625 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2626 }
2627#endif
2628 }
2629
2630 /*
2631 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2632 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2633 */
2634 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2635 {
2636 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2637 return VERR_VMX_INVALID_HOST_STATE;
2638 }
2639
2640 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2641#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2642 if (HMVMX_IS_64BIT_HOST_MODE())
2643 {
2644 /* We need the 64-bit TR base for hybrid darwin. */
2645 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2646 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2647 }
2648 else
2649#endif
2650 {
2651 uintptr_t uTRBase;
2652#if HC_ARCH_BITS == 64
2653 uTRBase = X86DESC64_BASE(pDesc);
2654
2655 /*
2656 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2657 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2658 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2659 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2660 *
2661 * [1] See Intel spec. 3.5 "System Descriptor Types".
2662 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2663 */
2664 Assert(pDesc->System.u4Type == 11);
2665 if ( pDesc->System.u16LimitLow != 0x67
2666 || pDesc->System.u4LimitHigh)
2667 {
2668 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2669 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2670
2671 /* Store the GDTR here as we need it while restoring TR. */
2672 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2673 }
2674#else
2675 uTRBase = X86DESC_BASE(pDesc);
2676#endif
2677 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2678 }
2679 AssertRCReturn(rc, rc);
2680
2681 /*
2682 * Host FS base and GS base.
2683 */
2684#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2685 if (HMVMX_IS_64BIT_HOST_MODE())
2686 {
2687 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2688 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2689 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2690 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2691
2692# if HC_ARCH_BITS == 64
2693 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2694 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2695 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2696 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2697 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2698# endif
2699 }
2700#endif
2701 return rc;
2702}
2703
2704
2705/**
2706 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2707 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2708 * the host after every successful VM exit.
2709 *
2710 * @returns VBox status code.
2711 * @param pVM Pointer to the VM.
2712 * @param pVCpu Pointer to the VMCPU.
2713 */
2714DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2715{
2716 AssertPtr(pVCpu);
2717 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2718
2719 int rc = VINF_SUCCESS;
2720#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2721 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2722 uint32_t cHostMsrs = 0;
2723 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2724
2725 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2726 {
2727 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2728
2729# if HC_ARCH_BITS == 64
2730 /* Paranoia. 64-bit code requires these bits to be set always. */
2731 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2732
2733 /*
2734 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2735 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2736 * some reason (e.g. allow transparent reads) we would activate the code below.
2737 */
2738# if 0
2739 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2740 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2741 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2742 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2743 if (CPUMIsGuestInLongMode(pVCpu))
2744 {
2745 uint64_t u64GuestEfer;
2746 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2747 AssertRC(rc);
2748
2749 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2750 {
2751 pHostMsr->u32Msr = MSR_K6_EFER;
2752 pHostMsr->u32Reserved = 0;
2753 pHostMsr->u64Value = u64HostEfer;
2754 pHostMsr++; cHostMsrs++;
2755 }
2756 }
2757# endif
2758# else /* HC_ARCH_BITS != 64 */
2759 pHostMsr->u32Msr = MSR_K6_EFER;
2760 pHostMsr->u32Reserved = 0;
2761# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2762 if (CPUMIsGuestInLongMode(pVCpu))
2763 {
2764 /* Must match the EFER value in our 64 bits switcher. */
2765 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2766 }
2767 else
2768# endif
2769 pHostMsr->u64Value = u64HostEfer;
2770 pHostMsr++; cHostMsrs++;
2771# endif /* HC_ARCH_BITS == 64 */
2772 }
2773
2774# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2775 if (HMVMX_IS_64BIT_HOST_MODE())
2776 {
2777 pHostMsr->u32Msr = MSR_K6_STAR;
2778 pHostMsr->u32Reserved = 0;
2779 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2780 pHostMsr++; cHostMsrs++;
2781 pHostMsr->u32Msr = MSR_K8_LSTAR;
2782 pHostMsr->u32Reserved = 0;
2783 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2784 pHostMsr++; cHostMsrs++;
2785 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2786 pHostMsr->u32Reserved = 0;
2787 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2788 pHostMsr++; cHostMsrs++;
2789 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2790 pHostMsr->u32Reserved = 0;
2791 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2792 pHostMsr++; cHostMsrs++;
2793 }
2794# endif
2795
2796 /* Host TSC AUX MSR must be restored since we always load/store guest TSC AUX MSR. */
2797 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2798 {
2799 pHostMsr->u32Msr = MSR_K8_TSC_AUX;
2800 pHostMsr->u32Reserved = 0;
2801 pHostMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
2802 pHostMsr++; cHostMsrs++;
2803 }
2804
2805 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2806 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2807 {
2808 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2809 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2810 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2811 }
2812
2813 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2814#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2815
2816 /*
2817 * Host Sysenter MSRs.
2818 */
2819 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2820 AssertRCReturn(rc, rc);
2821#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2822 if (HMVMX_IS_64BIT_HOST_MODE())
2823 {
2824 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2825 AssertRCReturn(rc, rc);
2826 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2827 }
2828 else
2829 {
2830 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2831 AssertRCReturn(rc, rc);
2832 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2833 }
2834#elif HC_ARCH_BITS == 32
2835 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2836 AssertRCReturn(rc, rc);
2837 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2838#else
2839 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2840 AssertRCReturn(rc, rc);
2841 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2842#endif
2843 AssertRCReturn(rc, rc);
2844
2845 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2846 * hmR0VmxSetupExitCtls() !! */
2847 return rc;
2848}
2849
2850
2851/**
2852 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2853 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2854 * controls".
2855 *
2856 * @returns VBox status code.
2857 * @param pVCpu Pointer to the VMCPU.
2858 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2859 * out-of-sync. Make sure to update the required fields
2860 * before using them.
2861 *
2862 * @remarks No-long-jump zone!!!
2863 */
2864DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2865{
2866 int rc = VINF_SUCCESS;
2867 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2868 {
2869 PVM pVM = pVCpu->CTX_SUFF(pVM);
2870 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2871 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2872
2873 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2874 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2875
2876 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2877 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2878 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2879 else
2880 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2881
2882 /*
2883 * The following should -not- be set (since we're not in SMM mode):
2884 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2885 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2886 */
2887
2888 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2889 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2890 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2891
2892 if ((val & zap) != val)
2893 {
2894 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2895 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2896 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2897 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2898 }
2899
2900 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2901 AssertRCReturn(rc, rc);
2902
2903 /* Update VCPU with the currently set VM-exit controls. */
2904 pVCpu->hm.s.vmx.u32EntryCtls = val;
2905 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2906 }
2907 return rc;
2908}
2909
2910
2911/**
2912 * Sets up the VM-exit controls in the VMCS.
2913 *
2914 * @returns VBox status code.
2915 * @param pVM Pointer to the VM.
2916 * @param pVCpu Pointer to the VMCPU.
2917 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2918 * out-of-sync. Make sure to update the required fields
2919 * before using them.
2920 *
2921 * @remarks requires EFER.
2922 */
2923DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2924{
2925 int rc = VINF_SUCCESS;
2926 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2927 {
2928 PVM pVM = pVCpu->CTX_SUFF(pVM);
2929 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2930 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2931
2932 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2933 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2934
2935 /*
2936 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2937 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2938 */
2939#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2940 if (HMVMX_IS_64BIT_HOST_MODE())
2941 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2942 else
2943 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2944#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2945 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2946 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2947 else
2948 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2949#endif
2950
2951 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2952 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2953
2954 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2955 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2956 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2957 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2958 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2959
2960 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2961 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2962
2963 if ((val & zap) != val)
2964 {
2965 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2966 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2967 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2968 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2969 }
2970
2971 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2972 AssertRCReturn(rc, rc);
2973
2974 /* Update VCPU with the currently set VM-exit controls. */
2975 pVCpu->hm.s.vmx.u32ExitCtls = val;
2976 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2977 }
2978 return rc;
2979}
2980
2981
2982/**
2983 * Loads the guest APIC and related state.
2984 *
2985 * @returns VBox status code.
2986 * @param pVM Pointer to the VM.
2987 * @param pVCpu Pointer to the VMCPU.
2988 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2989 * out-of-sync. Make sure to update the required fields
2990 * before using them.
2991 */
2992DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2993{
2994 int rc = VINF_SUCCESS;
2995 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2996 {
2997 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2998 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2999 {
3000 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3001
3002 bool fPendingIntr = false;
3003 uint8_t u8Tpr = 0;
3004 uint8_t u8PendingIntr = 0;
3005 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3006 AssertRCReturn(rc, rc);
3007
3008 /*
3009 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3010 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3011 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3012 * the interrupt when we VM-exit for other reasons.
3013 */
3014 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3015 uint32_t u32TprThreshold = 0;
3016 if (fPendingIntr)
3017 {
3018 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3019 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3020 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3021 if (u8PendingPriority <= u8TprPriority)
3022 u32TprThreshold = u8PendingPriority;
3023 else
3024 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3025 }
3026 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3027
3028 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3029 AssertRCReturn(rc, rc);
3030 }
3031
3032 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3033 }
3034 return rc;
3035}
3036
3037
3038/**
3039 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3040 *
3041 * @returns Guest's interruptibility-state.
3042 * @param pVCpu Pointer to the VMCPU.
3043 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3044 * out-of-sync. Make sure to update the required fields
3045 * before using them.
3046 *
3047 * @remarks No-long-jump zone!!!
3048 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3049 */
3050DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3051{
3052 /*
3053 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3054 * inhibit interrupts or clear any existing interrupt-inhibition.
3055 */
3056 uint32_t uIntrState = 0;
3057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3058 {
3059 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3060 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
3061 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3062 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3063 {
3064 /*
3065 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3066 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3067 */
3068 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3069 }
3070 else if (pMixedCtx->eflags.Bits.u1IF)
3071 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3072 else
3073 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3074 }
3075 return uIntrState;
3076}
3077
3078
3079/**
3080 * Loads the guest's interruptibility-state into the guest-state area in the
3081 * VMCS.
3082 *
3083 * @returns VBox status code.
3084 * @param pVCpu Pointer to the VMCPU.
3085 * @param uIntrState The interruptibility-state to set.
3086 */
3087static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3088{
3089 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3090 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3091 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3092 AssertRCReturn(rc, rc);
3093 return rc;
3094}
3095
3096
3097/**
3098 * Loads the guest's RIP into the guest-state area in the VMCS.
3099 *
3100 * @returns VBox status code.
3101 * @param pVCpu Pointer to the VMCPU.
3102 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3103 * out-of-sync. Make sure to update the required fields
3104 * before using them.
3105 *
3106 * @remarks No-long-jump zone!!!
3107 */
3108static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3109{
3110 int rc = VINF_SUCCESS;
3111 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3112 {
3113 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3114 AssertRCReturn(rc, rc);
3115
3116 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3117 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3118 }
3119 return rc;
3120}
3121
3122
3123/**
3124 * Loads the guest's RSP into the guest-state area in the VMCS.
3125 *
3126 * @returns VBox status code.
3127 * @param pVCpu Pointer to the VMCPU.
3128 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3129 * out-of-sync. Make sure to update the required fields
3130 * before using them.
3131 *
3132 * @remarks No-long-jump zone!!!
3133 */
3134static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3135{
3136 int rc = VINF_SUCCESS;
3137 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3138 {
3139 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3140 AssertRCReturn(rc, rc);
3141
3142 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3143 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3144 }
3145 return rc;
3146}
3147
3148
3149/**
3150 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3151 *
3152 * @returns VBox status code.
3153 * @param pVCpu Pointer to the VMCPU.
3154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3155 * out-of-sync. Make sure to update the required fields
3156 * before using them.
3157 *
3158 * @remarks No-long-jump zone!!!
3159 */
3160static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3161{
3162 int rc = VINF_SUCCESS;
3163 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3164 {
3165 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3166 Let us assert it as such and use 32-bit VMWRITE. */
3167 Assert(!(pMixedCtx->rflags.u64 >> 32));
3168 X86EFLAGS Eflags = pMixedCtx->eflags;
3169 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3170 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3171
3172 /*
3173 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3174 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3175 */
3176 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3177 {
3178 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3179 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3180 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3181 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3182 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3183 }
3184
3185 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3186 AssertRCReturn(rc, rc);
3187
3188 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3189 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3190 }
3191 return rc;
3192}
3193
3194
3195/**
3196 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3197 *
3198 * @returns VBox status code.
3199 * @param pVCpu Pointer to the VMCPU.
3200 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201 * out-of-sync. Make sure to update the required fields
3202 * before using them.
3203 *
3204 * @remarks No-long-jump zone!!!
3205 */
3206DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3207{
3208 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3209 AssertRCReturn(rc, rc);
3210 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3211 AssertRCReturn(rc, rc);
3212 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3213 AssertRCReturn(rc, rc);
3214 return rc;
3215}
3216
3217
3218/**
3219 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3220 * CR0 is partially shared with the host and we have to consider the FPU bits.
3221 *
3222 * @returns VBox status code.
3223 * @param pVM Pointer to the VM.
3224 * @param pVCpu Pointer to the VMCPU.
3225 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3226 * out-of-sync. Make sure to update the required fields
3227 * before using them.
3228 *
3229 * @remarks No-long-jump zone!!!
3230 */
3231static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3232{
3233 /*
3234 * Guest CR0.
3235 * Guest FPU.
3236 */
3237 int rc = VINF_SUCCESS;
3238 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3239 {
3240 Assert(!(pMixedCtx->cr0 >> 32));
3241 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3242 PVM pVM = pVCpu->CTX_SUFF(pVM);
3243
3244 /* The guest's view (read access) of its CR0 is unblemished. */
3245 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3246 AssertRCReturn(rc, rc);
3247 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3248
3249 /* Setup VT-x's view of the guest CR0. */
3250 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3251 if (pVM->hm.s.fNestedPaging)
3252 {
3253 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3254 {
3255 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3256 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3257 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3258 }
3259 else
3260 {
3261 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3262 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3263 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3264 }
3265
3266 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3267 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3268 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3269
3270 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3271 AssertRCReturn(rc, rc);
3272 }
3273 else
3274 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3275
3276 /*
3277 * Guest FPU bits.
3278 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3279 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3280 */
3281 u32GuestCR0 |= X86_CR0_NE;
3282 bool fInterceptNM = false;
3283 if (CPUMIsGuestFPUStateActive(pVCpu))
3284 {
3285 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3286 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3287 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3288 }
3289 else
3290 {
3291 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3292 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3293 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3294 }
3295
3296 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3297 bool fInterceptMF = false;
3298 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3299 fInterceptMF = true;
3300
3301 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3302 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3303 {
3304 Assert(PDMVmmDevHeapIsEnabled(pVM));
3305 Assert(pVM->hm.s.vmx.pRealModeTSS);
3306 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3307 fInterceptNM = true;
3308 fInterceptMF = true;
3309 }
3310 else
3311 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3312
3313 if (fInterceptNM)
3314 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3315 else
3316 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3317
3318 if (fInterceptMF)
3319 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3320 else
3321 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3322
3323 /* Additional intercepts for debugging, define these yourself explicitly. */
3324#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3325 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3326 | RT_BIT(X86_XCPT_BP)
3327 | RT_BIT(X86_XCPT_DB)
3328 | RT_BIT(X86_XCPT_DE)
3329 | RT_BIT(X86_XCPT_NM)
3330 | RT_BIT(X86_XCPT_UD)
3331 | RT_BIT(X86_XCPT_NP)
3332 | RT_BIT(X86_XCPT_SS)
3333 | RT_BIT(X86_XCPT_GP)
3334 | RT_BIT(X86_XCPT_PF)
3335 | RT_BIT(X86_XCPT_MF)
3336 ;
3337#elif defined(HMVMX_ALWAYS_TRAP_PF)
3338 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3339#endif
3340
3341 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3342
3343 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3344 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3345 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3346 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3347 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3348 else
3349 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3350
3351 u32GuestCR0 |= uSetCR0;
3352 u32GuestCR0 &= uZapCR0;
3353 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3354
3355 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3356 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3357 AssertRCReturn(rc, rc);
3358 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3359 AssertRCReturn(rc, rc);
3360 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3361
3362 /*
3363 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3364 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3365 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3366 */
3367 uint32_t u32CR0Mask = 0;
3368 u32CR0Mask = X86_CR0_PE
3369 | X86_CR0_NE
3370 | X86_CR0_WP
3371 | X86_CR0_PG
3372 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3373 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3374 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3375
3376 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3377 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3378 * and @bugref{6944}. */
3379#if 0
3380 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3381 u32CR0Mask &= ~X86_CR0_PE;
3382#endif
3383 if (pVM->hm.s.fNestedPaging)
3384 u32CR0Mask &= ~X86_CR0_WP;
3385
3386 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3387 if (fInterceptNM)
3388 {
3389 u32CR0Mask |= X86_CR0_TS
3390 | X86_CR0_MP;
3391 }
3392
3393 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3394 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3395 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3396 AssertRCReturn(rc, rc);
3397 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3398
3399 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3400 }
3401 return rc;
3402}
3403
3404
3405/**
3406 * Loads the guest control registers (CR3, CR4) into the guest-state area
3407 * in the VMCS.
3408 *
3409 * @returns VBox status code.
3410 * @param pVM Pointer to the VM.
3411 * @param pVCpu Pointer to the VMCPU.
3412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3413 * out-of-sync. Make sure to update the required fields
3414 * before using them.
3415 *
3416 * @remarks No-long-jump zone!!!
3417 */
3418static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3419{
3420 int rc = VINF_SUCCESS;
3421 PVM pVM = pVCpu->CTX_SUFF(pVM);
3422
3423 /*
3424 * Guest CR2.
3425 * It's always loaded in the assembler code. Nothing to do here.
3426 */
3427
3428 /*
3429 * Guest CR3.
3430 */
3431 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3432 {
3433 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3434 if (pVM->hm.s.fNestedPaging)
3435 {
3436 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3437
3438 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3439 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3440 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3441 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3442
3443 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3444 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3445 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3446
3447 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3448 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3449 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3450 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3451
3452 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3453 AssertRCReturn(rc, rc);
3454 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3455
3456 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3457 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3458 {
3459 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3460 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3461 {
3462 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3463 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3464 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3465 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3466 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3467 }
3468
3469 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3470 have Unrestricted Execution to handle the guest when it's not using paging. */
3471 GCPhysGuestCR3 = pMixedCtx->cr3;
3472 }
3473 else
3474 {
3475 /*
3476 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3477 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3478 * EPT takes care of translating it to host-physical addresses.
3479 */
3480 RTGCPHYS GCPhys;
3481 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3482 Assert(PDMVmmDevHeapIsEnabled(pVM));
3483
3484 /* We obtain it here every time as the guest could have relocated this PCI region. */
3485 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3486 AssertRCReturn(rc, rc);
3487
3488 GCPhysGuestCR3 = GCPhys;
3489 }
3490
3491 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3492 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3493 }
3494 else
3495 {
3496 /* Non-nested paging case, just use the hypervisor's CR3. */
3497 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3498
3499 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3500 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3501 }
3502 AssertRCReturn(rc, rc);
3503
3504 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3505 }
3506
3507 /*
3508 * Guest CR4.
3509 */
3510 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3511 {
3512 Assert(!(pMixedCtx->cr4 >> 32));
3513 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3514
3515 /* The guest's view of its CR4 is unblemished. */
3516 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3517 AssertRCReturn(rc, rc);
3518 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3519
3520 /* Setup VT-x's view of the guest CR4. */
3521 /*
3522 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3523 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3524 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3525 */
3526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3527 {
3528 Assert(pVM->hm.s.vmx.pRealModeTSS);
3529 Assert(PDMVmmDevHeapIsEnabled(pVM));
3530 u32GuestCR4 &= ~X86_CR4_VME;
3531 }
3532
3533 if (pVM->hm.s.fNestedPaging)
3534 {
3535 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3536 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3537 {
3538 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3539 u32GuestCR4 |= X86_CR4_PSE;
3540 /* Our identity mapping is a 32 bits page directory. */
3541 u32GuestCR4 &= ~X86_CR4_PAE;
3542 }
3543 /* else use guest CR4.*/
3544 }
3545 else
3546 {
3547 /*
3548 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3549 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3550 */
3551 switch (pVCpu->hm.s.enmShadowMode)
3552 {
3553 case PGMMODE_REAL: /* Real-mode. */
3554 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3555 case PGMMODE_32_BIT: /* 32-bit paging. */
3556 {
3557 u32GuestCR4 &= ~X86_CR4_PAE;
3558 break;
3559 }
3560
3561 case PGMMODE_PAE: /* PAE paging. */
3562 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3563 {
3564 u32GuestCR4 |= X86_CR4_PAE;
3565 break;
3566 }
3567
3568 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3569 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3570#ifdef VBOX_ENABLE_64_BITS_GUESTS
3571 break;
3572#endif
3573 default:
3574 AssertFailed();
3575 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3576 }
3577 }
3578
3579 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3580 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3581 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3582 u32GuestCR4 |= uSetCR4;
3583 u32GuestCR4 &= uZapCR4;
3584
3585 /* Write VT-x's view of the guest CR4 into the VMCS. */
3586 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3587 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3588 AssertRCReturn(rc, rc);
3589
3590 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3591 uint32_t u32CR4Mask = 0;
3592 u32CR4Mask = X86_CR4_VME
3593 | X86_CR4_PAE
3594 | X86_CR4_PGE
3595 | X86_CR4_PSE
3596 | X86_CR4_VMXE;
3597 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3598 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3599 AssertRCReturn(rc, rc);
3600
3601 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3602 }
3603 return rc;
3604}
3605
3606
3607/**
3608 * Loads the guest debug registers into the guest-state area in the VMCS.
3609 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3610 *
3611 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu Pointer to the VMCPU.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3624 return VINF_SUCCESS;
3625
3626#ifdef VBOX_STRICT
3627 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3628 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3629 {
3630 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3631 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3632 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3633 }
3634#endif
3635
3636 int rc;
3637 PVM pVM = pVCpu->CTX_SUFF(pVM);
3638 bool fInterceptDB = false;
3639 bool fInterceptMovDRx = false;
3640 if ( pVCpu->hm.s.fSingleInstruction
3641 || DBGFIsStepping(pVCpu))
3642 {
3643 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3644 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3645 {
3646 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3647 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3648 AssertRCReturn(rc, rc);
3649 Assert(fInterceptDB == false);
3650 }
3651 else
3652 {
3653 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3654 pVCpu->hm.s.fClearTrapFlag = true;
3655 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3656 fInterceptDB = true;
3657 }
3658 }
3659
3660 if ( fInterceptDB
3661 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3662 {
3663 /*
3664 * Use the combined guest and host DRx values found in the hypervisor
3665 * register set because the debugger has breakpoints active or someone
3666 * is single stepping on the host side without a monitor trap flag.
3667 *
3668 * Note! DBGF expects a clean DR6 state before executing guest code.
3669 */
3670#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3671 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3672 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3673 {
3674 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3675 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3676 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3677 }
3678 else
3679#endif
3680 if (!CPUMIsHyperDebugStateActive(pVCpu))
3681 {
3682 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3683 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3684 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3685 }
3686
3687 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3688 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3689 AssertRCReturn(rc, rc);
3690
3691 pVCpu->hm.s.fUsingHyperDR7 = true;
3692 fInterceptDB = true;
3693 fInterceptMovDRx = true;
3694 }
3695 else
3696 {
3697 /*
3698 * If the guest has enabled debug registers, we need to load them prior to
3699 * executing guest code so they'll trigger at the right time.
3700 */
3701 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3702 {
3703#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3704 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3705 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3706 {
3707 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3708 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3709 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3710 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3711 }
3712 else
3713#endif
3714 if (!CPUMIsGuestDebugStateActive(pVCpu))
3715 {
3716 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3717 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3718 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3719 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3720 }
3721 Assert(!fInterceptDB);
3722 Assert(!fInterceptMovDRx);
3723 }
3724 /*
3725 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3726 * must intercept #DB in order to maintain a correct DR6 guest value.
3727 */
3728#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3729 else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3730 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3731 || !CPUMIsGuestDebugStateActive(pVCpu))
3732#else
3733 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3734#endif
3735 {
3736 fInterceptMovDRx = true;
3737 fInterceptDB = true;
3738 }
3739
3740 /* Update guest DR7. */
3741 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3742 AssertRCReturn(rc, rc);
3743
3744 pVCpu->hm.s.fUsingHyperDR7 = false;
3745 }
3746
3747 /*
3748 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3749 */
3750 if (fInterceptDB)
3751 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3752 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3753 {
3754#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3755 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3756#endif
3757 }
3758 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3759 AssertRCReturn(rc, rc);
3760
3761 /*
3762 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3763 */
3764 if (fInterceptMovDRx)
3765 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3766 else
3767 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3768 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3769 AssertRCReturn(rc, rc);
3770
3771 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3772 return VINF_SUCCESS;
3773}
3774
3775
3776#ifdef VBOX_STRICT
3777/**
3778 * Strict function to validate segment registers.
3779 *
3780 * @remarks ASSUMES CR0 is up to date.
3781 */
3782static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3783{
3784 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3785 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3786 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3787 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3788 && ( !CPUMIsGuestInRealModeEx(pCtx)
3789 && !CPUMIsGuestInV86ModeEx(pCtx)))
3790 {
3791 /* Protected mode checks */
3792 /* CS */
3793 Assert(pCtx->cs.Attr.n.u1Present);
3794 Assert(!(pCtx->cs.Attr.u & 0xf00));
3795 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3796 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3797 || !(pCtx->cs.Attr.n.u1Granularity));
3798 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3799 || (pCtx->cs.Attr.n.u1Granularity));
3800 /* CS cannot be loaded with NULL in protected mode. */
3801 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3802 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3803 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3804 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3805 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3806 else
3807 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3808 /* SS */
3809 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3810 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3811 if ( !(pCtx->cr0 & X86_CR0_PE)
3812 || pCtx->cs.Attr.n.u4Type == 3)
3813 {
3814 Assert(!pCtx->ss.Attr.n.u2Dpl);
3815 }
3816 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3817 {
3818 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3819 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3820 Assert(pCtx->ss.Attr.n.u1Present);
3821 Assert(!(pCtx->ss.Attr.u & 0xf00));
3822 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3823 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3824 || !(pCtx->ss.Attr.n.u1Granularity));
3825 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3826 || (pCtx->ss.Attr.n.u1Granularity));
3827 }
3828 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3829 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3830 {
3831 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3832 Assert(pCtx->ds.Attr.n.u1Present);
3833 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3834 Assert(!(pCtx->ds.Attr.u & 0xf00));
3835 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3836 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3837 || !(pCtx->ds.Attr.n.u1Granularity));
3838 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3839 || (pCtx->ds.Attr.n.u1Granularity));
3840 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3841 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3842 }
3843 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3844 {
3845 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3846 Assert(pCtx->es.Attr.n.u1Present);
3847 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3848 Assert(!(pCtx->es.Attr.u & 0xf00));
3849 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3850 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3851 || !(pCtx->es.Attr.n.u1Granularity));
3852 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3853 || (pCtx->es.Attr.n.u1Granularity));
3854 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3855 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3856 }
3857 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3858 {
3859 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3860 Assert(pCtx->fs.Attr.n.u1Present);
3861 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3862 Assert(!(pCtx->fs.Attr.u & 0xf00));
3863 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3864 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3865 || !(pCtx->fs.Attr.n.u1Granularity));
3866 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3867 || (pCtx->fs.Attr.n.u1Granularity));
3868 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3869 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3870 }
3871 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3872 {
3873 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3874 Assert(pCtx->gs.Attr.n.u1Present);
3875 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3876 Assert(!(pCtx->gs.Attr.u & 0xf00));
3877 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3878 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3879 || !(pCtx->gs.Attr.n.u1Granularity));
3880 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3881 || (pCtx->gs.Attr.n.u1Granularity));
3882 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3883 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3884 }
3885 /* 64-bit capable CPUs. */
3886# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3887 Assert(!(pCtx->cs.u64Base >> 32));
3888 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3889 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3890 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3891# endif
3892 }
3893 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3894 || ( CPUMIsGuestInRealModeEx(pCtx)
3895 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3896 {
3897 /* Real and v86 mode checks. */
3898 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3899 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3900 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3901 {
3902 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3903 }
3904 else
3905 {
3906 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3907 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3908 }
3909
3910 /* CS */
3911 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3912 Assert(pCtx->cs.u32Limit == 0xffff);
3913 Assert(u32CSAttr == 0xf3);
3914 /* SS */
3915 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3916 Assert(pCtx->ss.u32Limit == 0xffff);
3917 Assert(u32SSAttr == 0xf3);
3918 /* DS */
3919 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3920 Assert(pCtx->ds.u32Limit == 0xffff);
3921 Assert(u32DSAttr == 0xf3);
3922 /* ES */
3923 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3924 Assert(pCtx->es.u32Limit == 0xffff);
3925 Assert(u32ESAttr == 0xf3);
3926 /* FS */
3927 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3928 Assert(pCtx->fs.u32Limit == 0xffff);
3929 Assert(u32FSAttr == 0xf3);
3930 /* GS */
3931 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3932 Assert(pCtx->gs.u32Limit == 0xffff);
3933 Assert(u32GSAttr == 0xf3);
3934 /* 64-bit capable CPUs. */
3935# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3936 Assert(!(pCtx->cs.u64Base >> 32));
3937 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3938 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3939 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3940# endif
3941 }
3942}
3943#endif /* VBOX_STRICT */
3944
3945
3946/**
3947 * Writes a guest segment register into the guest-state area in the VMCS.
3948 *
3949 * @returns VBox status code.
3950 * @param pVCpu Pointer to the VMCPU.
3951 * @param idxSel Index of the selector in the VMCS.
3952 * @param idxLimit Index of the segment limit in the VMCS.
3953 * @param idxBase Index of the segment base in the VMCS.
3954 * @param idxAccess Index of the access rights of the segment in the VMCS.
3955 * @param pSelReg Pointer to the segment selector.
3956 * @param pCtx Pointer to the guest-CPU context.
3957 *
3958 * @remarks No-long-jump zone!!!
3959 */
3960static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3961 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3962{
3963 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3964 AssertRCReturn(rc, rc);
3965 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3966 AssertRCReturn(rc, rc);
3967 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3968 AssertRCReturn(rc, rc);
3969
3970 uint32_t u32Access = pSelReg->Attr.u;
3971 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3972 {
3973 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3974 u32Access = 0xf3;
3975 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3976 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3977 }
3978 else
3979 {
3980 /*
3981 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3982 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3983 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3984 * loaded in protected-mode have their attribute as 0.
3985 */
3986 if (!u32Access)
3987 u32Access = X86DESCATTR_UNUSABLE;
3988 }
3989
3990 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3991 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3992 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3993
3994 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3995 AssertRCReturn(rc, rc);
3996 return rc;
3997}
3998
3999
4000/**
4001 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4002 * into the guest-state area in the VMCS.
4003 *
4004 * @returns VBox status code.
4005 * @param pVM Pointer to the VM.
4006 * @param pVCPU Pointer to the VMCPU.
4007 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4008 * out-of-sync. Make sure to update the required fields
4009 * before using them.
4010 *
4011 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4012 * @remarks No-long-jump zone!!!
4013 */
4014static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4015{
4016 int rc = VERR_INTERNAL_ERROR_5;
4017 PVM pVM = pVCpu->CTX_SUFF(pVM);
4018
4019 /*
4020 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4021 */
4022 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4023 {
4024 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4025 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4026 {
4027 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4028 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4029 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4030 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4031 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4032 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4033 }
4034
4035#ifdef VBOX_WITH_REM
4036 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4037 {
4038 Assert(pVM->hm.s.vmx.pRealModeTSS);
4039 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4040 if ( pVCpu->hm.s.vmx.fWasInRealMode
4041 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4042 {
4043 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4044 in real-mode (e.g. OpenBSD 4.0) */
4045 REMFlushTBs(pVM);
4046 Log4(("Load: Switch to protected mode detected!\n"));
4047 pVCpu->hm.s.vmx.fWasInRealMode = false;
4048 }
4049 }
4050#endif
4051 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4052 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
4053 AssertRCReturn(rc, rc);
4054 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4055 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
4056 AssertRCReturn(rc, rc);
4057 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4058 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
4059 AssertRCReturn(rc, rc);
4060 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4061 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
4062 AssertRCReturn(rc, rc);
4063 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4064 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
4065 AssertRCReturn(rc, rc);
4066 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4067 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
4068 AssertRCReturn(rc, rc);
4069
4070#ifdef VBOX_STRICT
4071 /* Validate. */
4072 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4073#endif
4074
4075 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4076 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4077 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4078 }
4079
4080 /*
4081 * Guest TR.
4082 */
4083 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4084 {
4085 /*
4086 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4087 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4088 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4089 */
4090 uint16_t u16Sel = 0;
4091 uint32_t u32Limit = 0;
4092 uint64_t u64Base = 0;
4093 uint32_t u32AccessRights = 0;
4094
4095 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4096 {
4097 u16Sel = pMixedCtx->tr.Sel;
4098 u32Limit = pMixedCtx->tr.u32Limit;
4099 u64Base = pMixedCtx->tr.u64Base;
4100 u32AccessRights = pMixedCtx->tr.Attr.u;
4101 }
4102 else
4103 {
4104 Assert(pVM->hm.s.vmx.pRealModeTSS);
4105 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4106
4107 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4108 RTGCPHYS GCPhys;
4109 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4110 AssertRCReturn(rc, rc);
4111
4112 X86DESCATTR DescAttr;
4113 DescAttr.u = 0;
4114 DescAttr.n.u1Present = 1;
4115 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4116
4117 u16Sel = 0;
4118 u32Limit = HM_VTX_TSS_SIZE;
4119 u64Base = GCPhys; /* in real-mode phys = virt. */
4120 u32AccessRights = DescAttr.u;
4121 }
4122
4123 /* Validate. */
4124 Assert(!(u16Sel & RT_BIT(2)));
4125 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4126 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4127 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4128 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4129 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4130 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4131 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4132 Assert( (u32Limit & 0xfff) == 0xfff
4133 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4134 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4135 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4136
4137 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4138 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4139 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4140 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4141
4142 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4143 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4144 }
4145
4146 /*
4147 * Guest GDTR.
4148 */
4149 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4150 {
4151 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4152 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4153
4154 /* Validate. */
4155 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4156
4157 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4158 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4159 }
4160
4161 /*
4162 * Guest LDTR.
4163 */
4164 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4165 {
4166 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4167 uint32_t u32Access = 0;
4168 if (!pMixedCtx->ldtr.Attr.u)
4169 u32Access = X86DESCATTR_UNUSABLE;
4170 else
4171 u32Access = pMixedCtx->ldtr.Attr.u;
4172
4173 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4174 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4175 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4176 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4177
4178 /* Validate. */
4179 if (!(u32Access & X86DESCATTR_UNUSABLE))
4180 {
4181 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4182 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4183 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4184 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4185 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4186 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4187 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4188 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4189 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4190 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4191 }
4192
4193 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4194 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4195 }
4196
4197 /*
4198 * Guest IDTR.
4199 */
4200 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4201 {
4202 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4203 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4204
4205 /* Validate. */
4206 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4207
4208 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4209 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4210 }
4211
4212 return VINF_SUCCESS;
4213}
4214
4215
4216/**
4217 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4218 * areas. These MSRs will automatically be loaded to the host CPU on every
4219 * successful VM entry and stored from the host CPU on every successful VM exit.
4220 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4221 *
4222 * @returns VBox status code.
4223 * @param pVCpu Pointer to the VMCPU.
4224 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4225 * out-of-sync. Make sure to update the required fields
4226 * before using them.
4227 *
4228 * @remarks No-long-jump zone!!!
4229 */
4230static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4231{
4232 AssertPtr(pVCpu);
4233 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4234
4235 /*
4236 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4237 */
4238 int rc = VINF_SUCCESS;
4239 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4240 {
4241#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4242 PVM pVM = pVCpu->CTX_SUFF(pVM);
4243 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4244 uint32_t cGuestMsrs = 0;
4245
4246 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4247 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4248 * when the guest really is in 64-bit mode. */
4249 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4250 if (fSupportsLongMode)
4251 {
4252 pGuestMsr->u32Msr = MSR_K8_LSTAR;
4253 pGuestMsr->u32Reserved = 0;
4254 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4255 pGuestMsr++; cGuestMsrs++;
4256 pGuestMsr->u32Msr = MSR_K6_STAR;
4257 pGuestMsr->u32Reserved = 0;
4258 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4259 pGuestMsr++; cGuestMsrs++;
4260 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4261 pGuestMsr->u32Reserved = 0;
4262 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4263 pGuestMsr++; cGuestMsrs++;
4264 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4265 pGuestMsr->u32Reserved = 0;
4266 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4267 pGuestMsr++; cGuestMsrs++;
4268 }
4269
4270 /*
4271 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4272 * load the guest's copy always (since the MSR bitmap allows passthru unconditionally).
4273 */
4274 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
4275 {
4276 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4277 pGuestMsr->u32Reserved = 0;
4278 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4279 AssertRCReturn(rc, rc);
4280 pGuestMsr++; cGuestMsrs++;
4281 }
4282
4283 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4284 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4285 {
4286 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4287 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4288 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4289 }
4290
4291 /* Update the VCPU's copy of the guest MSR count. */
4292 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4293 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4294 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4295#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4296
4297 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4298 }
4299
4300 /*
4301 * Guest Sysenter MSRs.
4302 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4303 * VM-exits on WRMSRs for these MSRs.
4304 */
4305 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4306 {
4307 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4308 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4309 }
4310
4311 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4312 {
4313 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4314 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4315 }
4316
4317 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4318 {
4319 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4320 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4321 }
4322
4323 return rc;
4324}
4325
4326
4327/**
4328 * Loads the guest activity state into the guest-state area in the VMCS.
4329 *
4330 * @returns VBox status code.
4331 * @param pVCpu Pointer to the VMCPU.
4332 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4333 * out-of-sync. Make sure to update the required fields
4334 * before using them.
4335 *
4336 * @remarks No-long-jump zone!!!
4337 */
4338static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4339{
4340 /** @todo See if we can make use of other states, e.g.
4341 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4342 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4343 {
4344 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4345 AssertRCReturn(rc, rc);
4346
4347 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4348 }
4349 return VINF_SUCCESS;
4350}
4351
4352
4353/**
4354 * Sets up the appropriate function to run guest code.
4355 *
4356 * @returns VBox status code.
4357 * @param pVCpu Pointer to the VMCPU.
4358 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4359 * out-of-sync. Make sure to update the required fields
4360 * before using them.
4361 *
4362 * @remarks No-long-jump zone!!!
4363 */
4364static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4365{
4366 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4367 {
4368#ifndef VBOX_ENABLE_64_BITS_GUESTS
4369 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4370#endif
4371 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4372#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4373 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4374 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4375 {
4376 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4377 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4378 }
4379#else
4380 /* 64-bit host or hybrid host. */
4381 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4382#endif
4383 }
4384 else
4385 {
4386 /* Guest is not in long mode, use the 32-bit handler. */
4387#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4388 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4389 {
4390 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4391 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4392 }
4393#else
4394 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4395#endif
4396 }
4397 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4398 return VINF_SUCCESS;
4399}
4400
4401
4402/**
4403 * Wrapper for running the guest code in VT-x.
4404 *
4405 * @returns VBox strict status code.
4406 * @param pVM Pointer to the VM.
4407 * @param pVCpu Pointer to the VMCPU.
4408 * @param pCtx Pointer to the guest-CPU context.
4409 *
4410 * @remarks No-long-jump zone!!!
4411 */
4412DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4413{
4414 /*
4415 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4416 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4417 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4418 */
4419 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4420 /** @todo Add stats for resume vs launch. */
4421#ifdef VBOX_WITH_KERNEL_USING_XMM
4422 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4423#else
4424 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4425#endif
4426}
4427
4428
4429/**
4430 * Reports world-switch error and dumps some useful debug info.
4431 *
4432 * @param pVM Pointer to the VM.
4433 * @param pVCpu Pointer to the VMCPU.
4434 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4435 * @param pCtx Pointer to the guest-CPU context.
4436 * @param pVmxTransient Pointer to the VMX transient structure (only
4437 * exitReason updated).
4438 */
4439static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4440{
4441 Assert(pVM);
4442 Assert(pVCpu);
4443 Assert(pCtx);
4444 Assert(pVmxTransient);
4445 HMVMX_ASSERT_PREEMPT_SAFE();
4446
4447 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4448 switch (rcVMRun)
4449 {
4450 case VERR_VMX_INVALID_VMXON_PTR:
4451 AssertFailed();
4452 break;
4453 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4454 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4455 {
4456 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4457 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4458 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4459 AssertRC(rc);
4460
4461 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4462 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4463 Cannot do it here as we may have been long preempted. */
4464
4465#ifdef VBOX_STRICT
4466 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4467 pVmxTransient->uExitReason));
4468 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4469 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4470 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4471 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4472 else
4473 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4474 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4475 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4476
4477 /* VMX control bits. */
4478 uint32_t u32Val;
4479 uint64_t u64Val;
4480 HMVMXHCUINTREG uHCReg;
4481 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4482 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4483 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4484 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4485 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4486 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4487 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4488 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4489 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4490 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4491 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4492 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4493 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4494 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4495 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4496 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4497 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4498 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4499 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4500 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4501 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4502 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4503 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4504 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4505 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4506 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4507 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4508 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4509 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4510 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4511 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4512 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4513 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4514 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4515 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4516 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4517 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4518 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4519 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4520 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4521 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4522 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4523
4524 /* Guest bits. */
4525 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4526 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4527 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4528 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4529 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4530 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4531 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4532 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4533
4534 /* Host bits. */
4535 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4536 Log4(("Host CR0 %#RHr\n", uHCReg));
4537 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4538 Log4(("Host CR3 %#RHr\n", uHCReg));
4539 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4540 Log4(("Host CR4 %#RHr\n", uHCReg));
4541
4542 RTGDTR HostGdtr;
4543 PCX86DESCHC pDesc;
4544 ASMGetGDTR(&HostGdtr);
4545 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4546 Log4(("Host CS %#08x\n", u32Val));
4547 if (u32Val < HostGdtr.cbGdt)
4548 {
4549 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4550 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4551 }
4552
4553 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4554 Log4(("Host DS %#08x\n", u32Val));
4555 if (u32Val < HostGdtr.cbGdt)
4556 {
4557 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4558 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4559 }
4560
4561 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4562 Log4(("Host ES %#08x\n", u32Val));
4563 if (u32Val < HostGdtr.cbGdt)
4564 {
4565 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4566 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4567 }
4568
4569 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4570 Log4(("Host FS %#08x\n", u32Val));
4571 if (u32Val < HostGdtr.cbGdt)
4572 {
4573 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4574 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4575 }
4576
4577 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4578 Log4(("Host GS %#08x\n", u32Val));
4579 if (u32Val < HostGdtr.cbGdt)
4580 {
4581 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4582 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4583 }
4584
4585 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4586 Log4(("Host SS %#08x\n", u32Val));
4587 if (u32Val < HostGdtr.cbGdt)
4588 {
4589 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4590 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4591 }
4592
4593 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4594 Log4(("Host TR %#08x\n", u32Val));
4595 if (u32Val < HostGdtr.cbGdt)
4596 {
4597 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4598 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4599 }
4600
4601 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4602 Log4(("Host TR Base %#RHv\n", uHCReg));
4603 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4604 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4605 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4606 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4607 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4608 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4609 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4610 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4611 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4612 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4613 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4614 Log4(("Host RSP %#RHv\n", uHCReg));
4615 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4616 Log4(("Host RIP %#RHv\n", uHCReg));
4617# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4618 if (HMVMX_IS_64BIT_HOST_MODE())
4619 {
4620 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4621 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4622 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4623 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4624 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4625 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4626 }
4627# endif
4628#endif /* VBOX_STRICT */
4629 break;
4630 }
4631
4632 default:
4633 /* Impossible */
4634 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4635 break;
4636 }
4637 NOREF(pVM);
4638}
4639
4640
4641#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4642#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4643# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4644#endif
4645#ifdef VBOX_STRICT
4646static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4647{
4648 switch (idxField)
4649 {
4650 case VMX_VMCS_GUEST_RIP:
4651 case VMX_VMCS_GUEST_RSP:
4652 case VMX_VMCS_GUEST_SYSENTER_EIP:
4653 case VMX_VMCS_GUEST_SYSENTER_ESP:
4654 case VMX_VMCS_GUEST_GDTR_BASE:
4655 case VMX_VMCS_GUEST_IDTR_BASE:
4656 case VMX_VMCS_GUEST_CS_BASE:
4657 case VMX_VMCS_GUEST_DS_BASE:
4658 case VMX_VMCS_GUEST_ES_BASE:
4659 case VMX_VMCS_GUEST_FS_BASE:
4660 case VMX_VMCS_GUEST_GS_BASE:
4661 case VMX_VMCS_GUEST_SS_BASE:
4662 case VMX_VMCS_GUEST_LDTR_BASE:
4663 case VMX_VMCS_GUEST_TR_BASE:
4664 case VMX_VMCS_GUEST_CR3:
4665 return true;
4666 }
4667 return false;
4668}
4669
4670static bool hmR0VmxIsValidReadField(uint32_t idxField)
4671{
4672 switch (idxField)
4673 {
4674 /* Read-only fields. */
4675 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4676 return true;
4677 }
4678 /* Remaining readable fields should also be writable. */
4679 return hmR0VmxIsValidWriteField(idxField);
4680}
4681#endif /* VBOX_STRICT */
4682
4683
4684/**
4685 * Executes the specified handler in 64-bit mode.
4686 *
4687 * @returns VBox status code.
4688 * @param pVM Pointer to the VM.
4689 * @param pVCpu Pointer to the VMCPU.
4690 * @param pCtx Pointer to the guest CPU context.
4691 * @param enmOp The operation to perform.
4692 * @param cbParam Number of parameters.
4693 * @param paParam Array of 32-bit parameters.
4694 */
4695VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4696 uint32_t *paParam)
4697{
4698 int rc, rc2;
4699 PHMGLOBALCPUINFO pCpu;
4700 RTHCPHYS HCPhysCpuPage;
4701 RTCCUINTREG uOldEflags;
4702
4703 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4704 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4705 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4706 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4707
4708#ifdef VBOX_STRICT
4709 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4710 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4711
4712 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4713 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4714#endif
4715
4716 /* Disable interrupts. */
4717 uOldEflags = ASMIntDisableFlags();
4718
4719#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4720 RTCPUID idHostCpu = RTMpCpuId();
4721 CPUMR0SetLApic(pVCpu, idHostCpu);
4722#endif
4723
4724 pCpu = HMR0GetCurrentCpu();
4725 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4726
4727 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4728 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4729
4730 /* Leave VMX Root Mode. */
4731 VMXDisable();
4732
4733 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4734
4735 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4736 CPUMSetHyperEIP(pVCpu, enmOp);
4737 for (int i = (int)cbParam - 1; i >= 0; i--)
4738 CPUMPushHyper(pVCpu, paParam[i]);
4739
4740 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4741
4742 /* Call the switcher. */
4743 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4744 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4745
4746 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4747 /* Make sure the VMX instructions don't cause #UD faults. */
4748 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4749
4750 /* Re-enter VMX Root Mode */
4751 rc2 = VMXEnable(HCPhysCpuPage);
4752 if (RT_FAILURE(rc2))
4753 {
4754 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4755 ASMSetFlags(uOldEflags);
4756 return rc2;
4757 }
4758
4759 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4760 AssertRC(rc2);
4761 Assert(!(ASMGetFlags() & X86_EFL_IF));
4762 ASMSetFlags(uOldEflags);
4763 return rc;
4764}
4765
4766
4767/**
4768 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4769 * supporting 64-bit guests.
4770 *
4771 * @returns VBox status code.
4772 * @param fResume Whether to VMLAUNCH or VMRESUME.
4773 * @param pCtx Pointer to the guest-CPU context.
4774 * @param pCache Pointer to the VMCS cache.
4775 * @param pVM Pointer to the VM.
4776 * @param pVCpu Pointer to the VMCPU.
4777 */
4778DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4779{
4780 uint32_t aParam[6];
4781 PHMGLOBALCPUINFO pCpu = NULL;
4782 RTHCPHYS HCPhysCpuPage = 0;
4783 int rc = VERR_INTERNAL_ERROR_5;
4784
4785 pCpu = HMR0GetCurrentCpu();
4786 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4787
4788#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4789 pCache->uPos = 1;
4790 pCache->interPD = PGMGetInterPaeCR3(pVM);
4791 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4792#endif
4793
4794#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4795 pCache->TestIn.HCPhysCpuPage = 0;
4796 pCache->TestIn.HCPhysVmcs = 0;
4797 pCache->TestIn.pCache = 0;
4798 pCache->TestOut.HCPhysVmcs = 0;
4799 pCache->TestOut.pCache = 0;
4800 pCache->TestOut.pCtx = 0;
4801 pCache->TestOut.eflags = 0;
4802#endif
4803
4804 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4805 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4806 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4807 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4808 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4809 aParam[5] = 0;
4810
4811#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4812 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4813 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4814#endif
4815 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4816
4817#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4818 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4819 Assert(pCtx->dr[4] == 10);
4820 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4821#endif
4822
4823#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4824 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4825 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4826 pVCpu->hm.s.vmx.HCPhysVmcs));
4827 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4828 pCache->TestOut.HCPhysVmcs));
4829 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4830 pCache->TestOut.pCache));
4831 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4832 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4833 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4834 pCache->TestOut.pCtx));
4835 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4836#endif
4837 return rc;
4838}
4839
4840
4841/**
4842 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4843 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4844 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4845 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4846 *
4847 * @returns VBox status code.
4848 * @param pVM Pointer to the VM.
4849 * @param pVCpu Pointer to the VMCPU.
4850 */
4851static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4852{
4853#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4854{ \
4855 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4856 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4857 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4858 ++cReadFields; \
4859}
4860
4861 AssertPtr(pVM);
4862 AssertPtr(pVCpu);
4863 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4864 uint32_t cReadFields = 0;
4865
4866 /*
4867 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4868 * and serve to indicate exceptions to the rules.
4869 */
4870
4871 /* Guest-natural selector base fields. */
4872#if 0
4873 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4874 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4875 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4876#endif
4877 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4878 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4879 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4880 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4881 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4882 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4883 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4884 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4885 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4886 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4887 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4888 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4889#if 0
4890 /* Unused natural width guest-state fields. */
4891 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4892 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4893#endif
4894 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4895 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4896
4897 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4898#if 0
4899 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4900 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4901 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4902 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4903 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4904 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4905 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4906 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4907 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4908#endif
4909
4910 /* Natural width guest-state fields. */
4911 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4912#if 0
4913 /* Currently unused field. */
4914 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4915#endif
4916
4917 if (pVM->hm.s.fNestedPaging)
4918 {
4919 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4920 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4921 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4922 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4923 }
4924 else
4925 {
4926 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4927 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4928 }
4929
4930#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4931 return VINF_SUCCESS;
4932}
4933
4934
4935/**
4936 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4937 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4938 * darwin, running 64-bit guests).
4939 *
4940 * @returns VBox status code.
4941 * @param pVCpu Pointer to the VMCPU.
4942 * @param idxField The VMCS field encoding.
4943 * @param u64Val 16, 32 or 64 bits value.
4944 */
4945VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4946{
4947 int rc;
4948 switch (idxField)
4949 {
4950 /*
4951 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4952 */
4953 /* 64-bit Control fields. */
4954 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4955 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4956 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4957 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4958 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4959 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4960 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4961 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4962 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4963 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4964 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4965 case VMX_VMCS64_CTRL_EPTP_FULL:
4966 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4967 /* 64-bit Guest-state fields. */
4968 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4969 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4970 case VMX_VMCS64_GUEST_PAT_FULL:
4971 case VMX_VMCS64_GUEST_EFER_FULL:
4972 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4973 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4974 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4975 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4976 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4977 /* 64-bit Host-state fields. */
4978 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4979 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4980 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4981 {
4982 rc = VMXWriteVmcs32(idxField, u64Val);
4983 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4984 break;
4985 }
4986
4987 /*
4988 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4989 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4990 */
4991 /* Natural-width Guest-state fields. */
4992 case VMX_VMCS_GUEST_CR3:
4993 case VMX_VMCS_GUEST_ES_BASE:
4994 case VMX_VMCS_GUEST_CS_BASE:
4995 case VMX_VMCS_GUEST_SS_BASE:
4996 case VMX_VMCS_GUEST_DS_BASE:
4997 case VMX_VMCS_GUEST_FS_BASE:
4998 case VMX_VMCS_GUEST_GS_BASE:
4999 case VMX_VMCS_GUEST_LDTR_BASE:
5000 case VMX_VMCS_GUEST_TR_BASE:
5001 case VMX_VMCS_GUEST_GDTR_BASE:
5002 case VMX_VMCS_GUEST_IDTR_BASE:
5003 case VMX_VMCS_GUEST_RSP:
5004 case VMX_VMCS_GUEST_RIP:
5005 case VMX_VMCS_GUEST_SYSENTER_ESP:
5006 case VMX_VMCS_GUEST_SYSENTER_EIP:
5007 {
5008 if (!(u64Val >> 32))
5009 {
5010 /* If this field is 64-bit, VT-x will zero out the top bits. */
5011 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5012 }
5013 else
5014 {
5015 /* Assert that only the 32->64 switcher case should ever come here. */
5016 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5017 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5018 }
5019 break;
5020 }
5021
5022 default:
5023 {
5024 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5025 rc = VERR_INVALID_PARAMETER;
5026 break;
5027 }
5028 }
5029 AssertRCReturn(rc, rc);
5030 return rc;
5031}
5032
5033
5034/**
5035 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5036 * hosts (except darwin) for 64-bit guests.
5037 *
5038 * @param pVCpu Pointer to the VMCPU.
5039 * @param idxField The VMCS field encoding.
5040 * @param u64Val 16, 32 or 64 bits value.
5041 */
5042VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5043{
5044 AssertPtr(pVCpu);
5045 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5046
5047 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5048 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5049
5050 /* Make sure there are no duplicates. */
5051 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5052 {
5053 if (pCache->Write.aField[i] == idxField)
5054 {
5055 pCache->Write.aFieldVal[i] = u64Val;
5056 return VINF_SUCCESS;
5057 }
5058 }
5059
5060 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5061 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5062 pCache->Write.cValidEntries++;
5063 return VINF_SUCCESS;
5064}
5065
5066/* Enable later when the assembly code uses these as callbacks. */
5067#if 0
5068/*
5069 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5070 *
5071 * @param pVCpu Pointer to the VMCPU.
5072 * @param pCache Pointer to the VMCS cache.
5073 *
5074 * @remarks No-long-jump zone!!!
5075 */
5076VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5077{
5078 AssertPtr(pCache);
5079 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5080 {
5081 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5082 AssertRC(rc);
5083 }
5084 pCache->Write.cValidEntries = 0;
5085}
5086
5087
5088/**
5089 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5090 *
5091 * @param pVCpu Pointer to the VMCPU.
5092 * @param pCache Pointer to the VMCS cache.
5093 *
5094 * @remarks No-long-jump zone!!!
5095 */
5096VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5097{
5098 AssertPtr(pCache);
5099 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5100 {
5101 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5102 AssertRC(rc);
5103 }
5104}
5105#endif
5106#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5107
5108
5109/**
5110 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5111 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5112 * timer.
5113 *
5114 * @returns VBox status code.
5115 * @param pVCpu Pointer to the VMCPU.
5116 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5117 * out-of-sync. Make sure to update the required fields
5118 * before using them.
5119 * @remarks No-long-jump zone!!!
5120 */
5121static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5122{
5123 int rc = VERR_INTERNAL_ERROR_5;
5124 bool fOffsettedTsc = false;
5125 PVM pVM = pVCpu->CTX_SUFF(pVM);
5126 if (pVM->hm.s.vmx.fUsePreemptTimer)
5127 {
5128 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5129
5130 /* Make sure the returned values have sane upper and lower boundaries. */
5131 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5132 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5133 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5134 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5135
5136 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5137 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5138 }
5139 else
5140 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5141
5142 if (fOffsettedTsc)
5143 {
5144 uint64_t u64CurTSC = ASMReadTSC();
5145 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5146 {
5147 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5148 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5149
5150 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5151 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5152 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5153 }
5154 else
5155 {
5156 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5157 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5158 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5159 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5160 }
5161 }
5162 else
5163 {
5164 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5165 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5166 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5167 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5168 }
5169}
5170
5171
5172/**
5173 * Determines if an exception is a contributory exception. Contributory
5174 * exceptions are ones which can cause double-faults. Page-fault is
5175 * intentionally not included here as it's a conditional contributory exception.
5176 *
5177 * @returns true if the exception is contributory, false otherwise.
5178 * @param uVector The exception vector.
5179 */
5180DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5181{
5182 switch (uVector)
5183 {
5184 case X86_XCPT_GP:
5185 case X86_XCPT_SS:
5186 case X86_XCPT_NP:
5187 case X86_XCPT_TS:
5188 case X86_XCPT_DE:
5189 return true;
5190 default:
5191 break;
5192 }
5193 return false;
5194}
5195
5196
5197/**
5198 * Sets an event as a pending event to be injected into the guest.
5199 *
5200 * @param pVCpu Pointer to the VMCPU.
5201 * @param u32IntInfo The VM-entry interruption-information field.
5202 * @param cbInstr The VM-entry instruction length in bytes (for software
5203 * interrupts, exceptions and privileged software
5204 * exceptions).
5205 * @param u32ErrCode The VM-entry exception error code.
5206 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5207 * page-fault.
5208 *
5209 * @remarks Statistics counter assumes this is a guest event being injected or
5210 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5211 * always incremented.
5212 */
5213DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5214 RTGCUINTPTR GCPtrFaultAddress)
5215{
5216 Assert(!pVCpu->hm.s.Event.fPending);
5217 pVCpu->hm.s.Event.fPending = true;
5218 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5219 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5220 pVCpu->hm.s.Event.cbInstr = cbInstr;
5221 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5222
5223 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5224}
5225
5226
5227/**
5228 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5229 *
5230 * @param pVCpu Pointer to the VMCPU.
5231 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5232 * out-of-sync. Make sure to update the required fields
5233 * before using them.
5234 */
5235DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5236{
5237 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5238 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5239 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5240 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5241}
5242
5243
5244/**
5245 * Handle a condition that occurred while delivering an event through the guest
5246 * IDT.
5247 *
5248 * @returns VBox status code (informational error codes included).
5249 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5250 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5251 * continue execution of the guest which will delivery the #DF.
5252 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5253 *
5254 * @param pVCpu Pointer to the VMCPU.
5255 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5256 * out-of-sync. Make sure to update the required fields
5257 * before using them.
5258 * @param pVmxTransient Pointer to the VMX transient structure.
5259 *
5260 * @remarks No-long-jump zone!!!
5261 */
5262static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5263{
5264 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5265 AssertRCReturn(rc, rc);
5266 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5267 {
5268 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5269 AssertRCReturn(rc, rc);
5270
5271 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5272 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5273 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5274
5275 typedef enum
5276 {
5277 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5278 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5279 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5280 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5281 } VMXREFLECTXCPT;
5282
5283 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5284 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5285 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5286 {
5287 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5288 {
5289 enmReflect = VMXREFLECTXCPT_XCPT;
5290#ifdef VBOX_STRICT
5291 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5292 && uExitVector == X86_XCPT_PF)
5293 {
5294 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5295 }
5296#endif
5297 if ( uExitVector == X86_XCPT_PF
5298 && uIdtVector == X86_XCPT_PF)
5299 {
5300 pVmxTransient->fVectoringPF = true;
5301 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5302 }
5303 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5304 && hmR0VmxIsContributoryXcpt(uExitVector)
5305 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5306 || uIdtVector == X86_XCPT_PF))
5307 {
5308 enmReflect = VMXREFLECTXCPT_DF;
5309 }
5310 else if (uIdtVector == X86_XCPT_DF)
5311 enmReflect = VMXREFLECTXCPT_TF;
5312 }
5313 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5314 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5315 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5316 {
5317 /*
5318 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5319 * (whatever they are) as they reoccur when restarting the instruction.
5320 */
5321 enmReflect = VMXREFLECTXCPT_XCPT;
5322 }
5323 }
5324 else
5325 {
5326 /*
5327 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5328 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5329 * original exception to the guest after handling the VM-exit.
5330 */
5331 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5332 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5333 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5334 {
5335 enmReflect = VMXREFLECTXCPT_XCPT;
5336 }
5337 }
5338
5339 switch (enmReflect)
5340 {
5341 case VMXREFLECTXCPT_XCPT:
5342 {
5343 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5344 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5345 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5346
5347 uint32_t u32ErrCode = 0;
5348 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5349 {
5350 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5351 AssertRCReturn(rc, rc);
5352 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5353 }
5354
5355 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5356 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5357 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5358 rc = VINF_SUCCESS;
5359 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5360 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5361
5362 break;
5363 }
5364
5365 case VMXREFLECTXCPT_DF:
5366 {
5367 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5368 rc = VINF_HM_DOUBLE_FAULT;
5369 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5370 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5371
5372 break;
5373 }
5374
5375 case VMXREFLECTXCPT_TF:
5376 {
5377 rc = VINF_EM_RESET;
5378 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5379 uExitVector));
5380 break;
5381 }
5382
5383 default:
5384 Assert(rc == VINF_SUCCESS);
5385 break;
5386 }
5387 }
5388 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5389 return rc;
5390}
5391
5392
5393/**
5394 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5395 *
5396 * @returns VBox status code.
5397 * @param pVCpu Pointer to the VMCPU.
5398 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5399 * out-of-sync. Make sure to update the required fields
5400 * before using them.
5401 *
5402 * @remarks No-long-jump zone!!!
5403 */
5404static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5405{
5406 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5407 {
5408 uint32_t uVal = 0;
5409 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5410 AssertRCReturn(rc, rc);
5411
5412 uint32_t uShadow = 0;
5413 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5414 AssertRCReturn(rc, rc);
5415
5416 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5417 CPUMSetGuestCR0(pVCpu, uVal);
5418 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5419 }
5420 return VINF_SUCCESS;
5421}
5422
5423
5424/**
5425 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5426 *
5427 * @returns VBox status code.
5428 * @param pVCpu Pointer to the VMCPU.
5429 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5430 * out-of-sync. Make sure to update the required fields
5431 * before using them.
5432 *
5433 * @remarks No-long-jump zone!!!
5434 */
5435static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5436{
5437 int rc = VINF_SUCCESS;
5438 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5439 {
5440 uint32_t uVal = 0;
5441 uint32_t uShadow = 0;
5442 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5443 AssertRCReturn(rc, rc);
5444 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5445 AssertRCReturn(rc, rc);
5446
5447 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5448 CPUMSetGuestCR4(pVCpu, uVal);
5449 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5450 }
5451 return rc;
5452}
5453
5454
5455/**
5456 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5457 *
5458 * @returns VBox status code.
5459 * @param pVCpu Pointer to the VMCPU.
5460 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5461 * out-of-sync. Make sure to update the required fields
5462 * before using them.
5463 *
5464 * @remarks No-long-jump zone!!!
5465 */
5466static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5467{
5468 int rc = VINF_SUCCESS;
5469 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5470 {
5471 uint64_t u64Val = 0;
5472 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5473 AssertRCReturn(rc, rc);
5474
5475 pMixedCtx->rip = u64Val;
5476 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5477 }
5478 return rc;
5479}
5480
5481
5482/**
5483 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5484 *
5485 * @returns VBox status code.
5486 * @param pVCpu Pointer to the VMCPU.
5487 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5488 * out-of-sync. Make sure to update the required fields
5489 * before using them.
5490 *
5491 * @remarks No-long-jump zone!!!
5492 */
5493static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5494{
5495 int rc = VINF_SUCCESS;
5496 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5497 {
5498 uint64_t u64Val = 0;
5499 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5500 AssertRCReturn(rc, rc);
5501
5502 pMixedCtx->rsp = u64Val;
5503 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5504 }
5505 return rc;
5506}
5507
5508
5509/**
5510 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5511 *
5512 * @returns VBox status code.
5513 * @param pVCpu Pointer to the VMCPU.
5514 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5515 * out-of-sync. Make sure to update the required fields
5516 * before using them.
5517 *
5518 * @remarks No-long-jump zone!!!
5519 */
5520static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5521{
5522 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5523 {
5524 uint32_t uVal = 0;
5525 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5526 AssertRCReturn(rc, rc);
5527
5528 pMixedCtx->eflags.u32 = uVal;
5529 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5530 {
5531 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5532 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5533
5534 pMixedCtx->eflags.Bits.u1VM = 0;
5535 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5536 }
5537
5538 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5539 }
5540 return VINF_SUCCESS;
5541}
5542
5543
5544/**
5545 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5546 * guest-CPU context.
5547 */
5548DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5549{
5550 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5551 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5552 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5553 return rc;
5554}
5555
5556
5557/**
5558 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5559 * from the guest-state area in the VMCS.
5560 *
5561 * @param pVCpu Pointer to the VMCPU.
5562 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5563 * out-of-sync. Make sure to update the required fields
5564 * before using them.
5565 *
5566 * @remarks No-long-jump zone!!!
5567 */
5568static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5569{
5570 uint32_t uIntrState = 0;
5571 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5572 AssertRC(rc);
5573
5574 if (!uIntrState)
5575 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5576 else
5577 {
5578 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5579 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5580 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5581 AssertRC(rc);
5582 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5583 AssertRC(rc);
5584
5585 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5586 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5587 }
5588}
5589
5590
5591/**
5592 * Saves the guest's activity state.
5593 *
5594 * @returns VBox status code.
5595 * @param pVCpu Pointer to the VMCPU.
5596 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5597 * out-of-sync. Make sure to update the required fields
5598 * before using them.
5599 *
5600 * @remarks No-long-jump zone!!!
5601 */
5602static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5603{
5604 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5605 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5606 return VINF_SUCCESS;
5607}
5608
5609
5610/**
5611 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5612 * the current VMCS into the guest-CPU context.
5613 *
5614 * @returns VBox status code.
5615 * @param pVCpu Pointer to the VMCPU.
5616 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5617 * out-of-sync. Make sure to update the required fields
5618 * before using them.
5619 *
5620 * @remarks No-long-jump zone!!!
5621 */
5622static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5623{
5624 int rc = VINF_SUCCESS;
5625 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5626 {
5627 uint32_t u32Val = 0;
5628 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5629 pMixedCtx->SysEnter.cs = u32Val;
5630 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5631 }
5632
5633 uint64_t u64Val = 0;
5634 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5635 {
5636 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5637 pMixedCtx->SysEnter.eip = u64Val;
5638 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5639 }
5640 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5641 {
5642 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5643 pMixedCtx->SysEnter.esp = u64Val;
5644 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5645 }
5646 return rc;
5647}
5648
5649
5650/**
5651 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5652 * context.
5653 *
5654 * @returns VBox status code.
5655 * @param pVCpu Pointer to the VMCPU.
5656 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5657 * out-of-sync. Make sure to update the required fields
5658 * before using them.
5659 *
5660 * @remarks No-long-jump zone!!!
5661 */
5662static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5663{
5664 int rc = VINF_SUCCESS;
5665 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5666 {
5667 uint64_t u64Val = 0;
5668 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5669 pMixedCtx->fs.u64Base = u64Val;
5670 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5671 }
5672 return rc;
5673}
5674
5675
5676/**
5677 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5678 * context.
5679 *
5680 * @returns VBox status code.
5681 * @param pVCpu Pointer to the VMCPU.
5682 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5683 * out-of-sync. Make sure to update the required fields
5684 * before using them.
5685 *
5686 * @remarks No-long-jump zone!!!
5687 */
5688static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5689{
5690 int rc = VINF_SUCCESS;
5691 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5692 {
5693 uint64_t u64Val = 0;
5694 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5695 pMixedCtx->gs.u64Base = u64Val;
5696 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5697 }
5698 return rc;
5699}
5700
5701
5702/**
5703 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5704 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5705 * and TSC_AUX.
5706 *
5707 * @returns VBox status code.
5708 * @param pVCpu Pointer to the VMCPU.
5709 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5710 * out-of-sync. Make sure to update the required fields
5711 * before using them.
5712 *
5713 * @remarks No-long-jump zone!!!
5714 */
5715static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5716{
5717 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5718 return VINF_SUCCESS;
5719
5720#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5721 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5722 {
5723 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5724 pMsr += i;
5725 switch (pMsr->u32Msr)
5726 {
5727 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5728 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5729 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5730 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5731 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5732 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5733 default:
5734 {
5735 AssertFailed();
5736 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5737 }
5738 }
5739 }
5740#endif
5741
5742 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5743 return VINF_SUCCESS;
5744}
5745
5746
5747/**
5748 * Saves the guest control registers from the current VMCS into the guest-CPU
5749 * context.
5750 *
5751 * @returns VBox status code.
5752 * @param pVCpu Pointer to the VMCPU.
5753 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5754 * out-of-sync. Make sure to update the required fields
5755 * before using them.
5756 *
5757 * @remarks No-long-jump zone!!!
5758 */
5759static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5760{
5761 /* Guest CR0. Guest FPU. */
5762 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5763 AssertRCReturn(rc, rc);
5764
5765 /* Guest CR4. */
5766 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5767 AssertRCReturn(rc, rc);
5768
5769 /* Guest CR2 - updated always during the world-switch or in #PF. */
5770 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5771 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5772 {
5773 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5774 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5775
5776 PVM pVM = pVCpu->CTX_SUFF(pVM);
5777 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5778 || ( pVM->hm.s.fNestedPaging
5779 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5780 {
5781 uint64_t u64Val = 0;
5782 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5783 if (pMixedCtx->cr3 != u64Val)
5784 {
5785 CPUMSetGuestCR3(pVCpu, u64Val);
5786 if (VMMRZCallRing3IsEnabled(pVCpu))
5787 {
5788 PGMUpdateCR3(pVCpu, u64Val);
5789 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5790 }
5791 else
5792 {
5793 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5794 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5795 }
5796 }
5797
5798 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5799 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5800 {
5801 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5802 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5803 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5804 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5805
5806 if (VMMRZCallRing3IsEnabled(pVCpu))
5807 {
5808 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5809 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5810 }
5811 else
5812 {
5813 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5814 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5815 }
5816 }
5817 }
5818
5819 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5820 }
5821
5822 /*
5823 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5824 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5825 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5826 *
5827 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5828 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5829 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5830 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5831 *
5832 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5833 */
5834 if (VMMRZCallRing3IsEnabled(pVCpu))
5835 {
5836 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5837 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5838
5839 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5840 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5841
5842 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5843 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5844 }
5845
5846 return rc;
5847}
5848
5849
5850/**
5851 * Reads a guest segment register from the current VMCS into the guest-CPU
5852 * context.
5853 *
5854 * @returns VBox status code.
5855 * @param pVCpu Pointer to the VMCPU.
5856 * @param idxSel Index of the selector in the VMCS.
5857 * @param idxLimit Index of the segment limit in the VMCS.
5858 * @param idxBase Index of the segment base in the VMCS.
5859 * @param idxAccess Index of the access rights of the segment in the VMCS.
5860 * @param pSelReg Pointer to the segment selector.
5861 *
5862 * @remarks No-long-jump zone!!!
5863 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5864 * macro as that takes care of whether to read from the VMCS cache or
5865 * not.
5866 */
5867DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5868 PCPUMSELREG pSelReg)
5869{
5870 uint32_t u32Val = 0;
5871 int rc = VMXReadVmcs32(idxSel, &u32Val);
5872 AssertRCReturn(rc, rc);
5873 pSelReg->Sel = (uint16_t)u32Val;
5874 pSelReg->ValidSel = (uint16_t)u32Val;
5875 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5876
5877 rc = VMXReadVmcs32(idxLimit, &u32Val);
5878 AssertRCReturn(rc, rc);
5879 pSelReg->u32Limit = u32Val;
5880
5881 uint64_t u64Val = 0;
5882 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5883 AssertRCReturn(rc, rc);
5884 pSelReg->u64Base = u64Val;
5885
5886 rc = VMXReadVmcs32(idxAccess, &u32Val);
5887 AssertRCReturn(rc, rc);
5888 pSelReg->Attr.u = u32Val;
5889
5890 /*
5891 * If VT-x marks the segment as unusable, most other bits remain undefined:
5892 * - For CS the L, D and G bits have meaning.
5893 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5894 * - For the remaining data segments no bits are defined.
5895 *
5896 * The present bit and the unusable bit has been observed to be set at the
5897 * same time (the selector was supposed to invalid as we started executing
5898 * a V8086 interrupt in ring-0).
5899 *
5900 * What should be important for the rest of the VBox code that the P bit is
5901 * cleared. Some of the other VBox code recognizes the unusable bit, but
5902 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5903 * safe side here, we'll strip off P and other bits we don't care about. If
5904 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5905 *
5906 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5907 */
5908 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5909 {
5910 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5911
5912 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5913 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5914 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5915
5916 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5917#ifdef DEBUG_bird
5918 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5919 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5920 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5921#endif
5922 }
5923 return VINF_SUCCESS;
5924}
5925
5926
5927#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5928# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5929 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5930 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5931#else
5932# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5933 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5934 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5935#endif
5936
5937
5938/**
5939 * Saves the guest segment registers from the current VMCS into the guest-CPU
5940 * context.
5941 *
5942 * @returns VBox status code.
5943 * @param pVCpu Pointer to the VMCPU.
5944 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5945 * out-of-sync. Make sure to update the required fields
5946 * before using them.
5947 *
5948 * @remarks No-long-jump zone!!!
5949 */
5950static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5951{
5952 /* Guest segment registers. */
5953 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5954 {
5955 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5956 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5957 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5958 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5959 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5960 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5961 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5962
5963 /* Restore segment attributes for real-on-v86 mode hack. */
5964 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5965 {
5966 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5967 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5968 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5969 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5970 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5971 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5972 }
5973 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5974 }
5975
5976 return VINF_SUCCESS;
5977}
5978
5979
5980/**
5981 * Saves the guest descriptor table registers and task register from the current
5982 * VMCS into the guest-CPU context.
5983 *
5984 * @returns VBox status code.
5985 * @param pVCpu Pointer to the VMCPU.
5986 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5987 * out-of-sync. Make sure to update the required fields
5988 * before using them.
5989 *
5990 * @remarks No-long-jump zone!!!
5991 */
5992static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5993{
5994 int rc = VINF_SUCCESS;
5995
5996 /* Guest LDTR. */
5997 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5998 {
5999 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6000 AssertRCReturn(rc, rc);
6001 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
6002 }
6003
6004 /* Guest GDTR. */
6005 uint64_t u64Val = 0;
6006 uint32_t u32Val = 0;
6007 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6008 {
6009 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6010 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6011 pMixedCtx->gdtr.pGdt = u64Val;
6012 pMixedCtx->gdtr.cbGdt = u32Val;
6013 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
6014 }
6015
6016 /* Guest IDTR. */
6017 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6018 {
6019 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6020 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6021 pMixedCtx->idtr.pIdt = u64Val;
6022 pMixedCtx->idtr.cbIdt = u32Val;
6023 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
6024 }
6025
6026 /* Guest TR. */
6027 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6028 {
6029 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6030 AssertRCReturn(rc, rc);
6031
6032 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6033 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6034 {
6035 rc = VMXLOCAL_READ_SEG(TR, tr);
6036 AssertRCReturn(rc, rc);
6037 }
6038 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
6039 }
6040 return rc;
6041}
6042
6043#undef VMXLOCAL_READ_SEG
6044
6045
6046/**
6047 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6048 * context.
6049 *
6050 * @returns VBox status code.
6051 * @param pVCpu Pointer to the VMCPU.
6052 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6053 * out-of-sync. Make sure to update the required fields
6054 * before using them.
6055 *
6056 * @remarks No-long-jump zone!!!
6057 */
6058static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6059{
6060 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6061 {
6062 if (!pVCpu->hm.s.fUsingHyperDR7)
6063 {
6064 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6065 uint32_t u32Val;
6066 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6067 pMixedCtx->dr[7] = u32Val;
6068 }
6069
6070 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
6071 }
6072 return VINF_SUCCESS;
6073}
6074
6075
6076/**
6077 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6078 *
6079 * @returns VBox status code.
6080 * @param pVCpu Pointer to the VMCPU.
6081 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6082 * out-of-sync. Make sure to update the required fields
6083 * before using them.
6084 *
6085 * @remarks No-long-jump zone!!!
6086 */
6087static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6088{
6089 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6090 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
6091 return VINF_SUCCESS;
6092}
6093
6094
6095/**
6096 * Saves the entire guest state from the currently active VMCS into the
6097 * guest-CPU context. This essentially VMREADs all guest-data.
6098 *
6099 * @returns VBox status code.
6100 * @param pVCpu Pointer to the VMCPU.
6101 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6102 * out-of-sync. Make sure to update the required fields
6103 * before using them.
6104 */
6105static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6106{
6107 Assert(pVCpu);
6108 Assert(pMixedCtx);
6109
6110 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6111 return VINF_SUCCESS;
6112
6113 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6114 again on the ring-3 callback path, there is no real need to. */
6115 if (VMMRZCallRing3IsEnabled(pVCpu))
6116 VMMR0LogFlushDisable(pVCpu);
6117 else
6118 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6119 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6120
6121 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6122 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6123
6124 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6125 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6126
6127 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6128 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6129
6130 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6131 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6132
6133 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6134 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6135
6136 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6137 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6138
6139 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
6140 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6141
6142 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
6143 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6144
6145 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6146 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6147
6148 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6149 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6150
6151 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6152 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6153
6154 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6155 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6156
6157 if (VMMRZCallRing3IsEnabled(pVCpu))
6158 VMMR0LogFlushEnable(pVCpu);
6159
6160 return rc;
6161}
6162
6163
6164/**
6165 * Check per-VM and per-VCPU force flag actions that require us to go back to
6166 * ring-3 for one reason or another.
6167 *
6168 * @returns VBox status code (information status code included).
6169 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6170 * ring-3.
6171 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6172 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6173 * interrupts)
6174 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6175 * all EMTs to be in ring-3.
6176 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6177 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6178 * to the EM loop.
6179 *
6180 * @param pVM Pointer to the VM.
6181 * @param pVCpu Pointer to the VMCPU.
6182 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6183 * out-of-sync. Make sure to update the required fields
6184 * before using them.
6185 */
6186static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6187{
6188 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6189
6190 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6191 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6192 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6193 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6194 {
6195 /* We need the control registers now, make sure the guest-CPU context is updated. */
6196 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6197 AssertRCReturn(rc3, rc3);
6198
6199 /* Pending HM CR3 sync. */
6200 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6201 {
6202 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6203 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6204 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6205 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6206 }
6207
6208 /* Pending HM PAE PDPEs. */
6209 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6210 {
6211 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6212 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6213 }
6214
6215 /* Pending PGM C3 sync. */
6216 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6217 {
6218 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6219 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6220 if (rc2 != VINF_SUCCESS)
6221 {
6222 AssertRC(rc2);
6223 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6224 return rc2;
6225 }
6226 }
6227
6228 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6229 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6230 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6231 {
6232 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6233 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6234 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6235 return rc2;
6236 }
6237
6238 /* Pending VM request packets, such as hardware interrupts. */
6239 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6240 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6241 {
6242 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6243 return VINF_EM_PENDING_REQUEST;
6244 }
6245
6246 /* Pending PGM pool flushes. */
6247 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6248 {
6249 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6250 return VINF_PGM_POOL_FLUSH_PENDING;
6251 }
6252
6253 /* Pending DMA requests. */
6254 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6255 {
6256 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6257 return VINF_EM_RAW_TO_R3;
6258 }
6259 }
6260
6261 return VINF_SUCCESS;
6262}
6263
6264
6265/**
6266 * Converts any TRPM trap into a pending HM event. This is typically used when
6267 * entering from ring-3 (not longjmp returns).
6268 *
6269 * @param pVCpu Pointer to the VMCPU.
6270 */
6271static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6272{
6273 Assert(TRPMHasTrap(pVCpu));
6274 Assert(!pVCpu->hm.s.Event.fPending);
6275
6276 uint8_t uVector;
6277 TRPMEVENT enmTrpmEvent;
6278 RTGCUINT uErrCode;
6279 RTGCUINTPTR GCPtrFaultAddress;
6280 uint8_t cbInstr;
6281
6282 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6283 AssertRC(rc);
6284
6285 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6286 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6287 if (enmTrpmEvent == TRPM_TRAP)
6288 {
6289 switch (uVector)
6290 {
6291 case X86_XCPT_BP:
6292 case X86_XCPT_OF:
6293 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6294 break;
6295
6296 case X86_XCPT_PF:
6297 case X86_XCPT_DF:
6298 case X86_XCPT_TS:
6299 case X86_XCPT_NP:
6300 case X86_XCPT_SS:
6301 case X86_XCPT_GP:
6302 case X86_XCPT_AC:
6303 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6304 /* no break! */
6305 default:
6306 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6307 break;
6308 }
6309 }
6310 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6311 {
6312 if (uVector == X86_XCPT_NMI)
6313 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6314 else
6315 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6316 }
6317 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6318 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6319 else
6320 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6321
6322 rc = TRPMResetTrap(pVCpu);
6323 AssertRC(rc);
6324 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6325 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6326
6327 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6328 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6329}
6330
6331
6332/**
6333 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6334 * VT-x to execute any instruction.
6335 *
6336 * @param pvCpu Pointer to the VMCPU.
6337 */
6338static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6339{
6340 Assert(pVCpu->hm.s.Event.fPending);
6341
6342 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6343 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6344 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6345 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6346
6347 /* If a trap was already pending, we did something wrong! */
6348 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6349
6350 TRPMEVENT enmTrapType;
6351 switch (uVectorType)
6352 {
6353 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6354 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6355 enmTrapType = TRPM_HARDWARE_INT;
6356 break;
6357
6358 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6359 enmTrapType = TRPM_SOFTWARE_INT;
6360 break;
6361
6362 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6363 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6364 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6365 enmTrapType = TRPM_TRAP;
6366 break;
6367
6368 default:
6369 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6370 enmTrapType = TRPM_32BIT_HACK;
6371 break;
6372 }
6373
6374 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6375
6376 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6377 AssertRC(rc);
6378
6379 if (fErrorCodeValid)
6380 TRPMSetErrorCode(pVCpu, uErrorCode);
6381
6382 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6383 && uVector == X86_XCPT_PF)
6384 {
6385 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6386 }
6387 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6388 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6389 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6390 {
6391 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6392 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6393 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6394 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6395 }
6396 pVCpu->hm.s.Event.fPending = false;
6397}
6398
6399
6400/**
6401 * Does the necessary state syncing before returning to ring-3 for any reason
6402 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6403 *
6404 * @returns VBox status code.
6405 * @param pVM Pointer to the VM.
6406 * @param pVCpu Pointer to the VMCPU.
6407 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6408 * be out-of-sync. Make sure to update the required
6409 * fields before using them.
6410 * @param fSaveGuestState Whether to save the guest state or not.
6411 *
6412 * @remarks If you modify code here, make sure to check whether
6413 * hmR0VmxCallRing3Callback() needs to be updated too.
6414 * @remarks No-long-jmp zone!!!
6415 */
6416static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6417{
6418 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6419 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6420
6421 RTCPUID idCpu = RTMpCpuId();
6422 Log4Func(("HostCpuId=%u\n", idCpu));
6423
6424 /* Save the guest state if necessary. */
6425 if ( fSaveGuestState
6426 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6427 {
6428 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6429 AssertRCReturn(rc, rc);
6430 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6431 }
6432
6433 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6434 if (CPUMIsGuestFPUStateActive(pVCpu))
6435 {
6436 /* We shouldn't reload CR0 without saving it first. */
6437 if (!fSaveGuestState)
6438 {
6439 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6440 AssertRCReturn(rc, rc);
6441 }
6442 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6443 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6444 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6445 }
6446
6447 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6448#ifdef VBOX_STRICT
6449 if (CPUMIsHyperDebugStateActive(pVCpu))
6450 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6451#endif
6452 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6453 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6454 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6455 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6456
6457#if HC_ARCH_BITS == 64
6458 /* Restore host-state bits that VT-x only restores partially. */
6459 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6460 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6461 {
6462 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6463 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6464 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6465 }
6466#endif
6467
6468 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6469 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6470 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6471 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6472 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6473 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6474 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6475 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6476
6477 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6478
6479 /** @todo This kinda defeats the purpose of having preemption hooks.
6480 * The problem is, deregistering the hooks should be moved to a place that
6481 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6482 * context.
6483 */
6484 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6485 {
6486 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6487 AssertRCReturn(rc, rc);
6488
6489 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6490 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6491 }
6492 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6493 NOREF(idCpu);
6494
6495 return VINF_SUCCESS;
6496}
6497
6498
6499/**
6500 * Leaves the VT-x session.
6501 *
6502 * @returns VBox status code.
6503 * @param pVM Pointer to the VM.
6504 * @param pVCpu Pointer to the VMCPU.
6505 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6506 * out-of-sync. Make sure to update the required fields
6507 * before using them.
6508 *
6509 * @remarks No-long-jmp zone!!!
6510 */
6511DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6512{
6513 HM_DISABLE_PREEMPT_IF_NEEDED();
6514 HMVMX_ASSERT_CPU_SAFE();
6515 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6516 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6517
6518 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6519 and done this from the VMXR0ThreadCtxCallback(). */
6520 if (!pVCpu->hm.s.fLeaveDone)
6521 {
6522 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6523 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6524 pVCpu->hm.s.fLeaveDone = true;
6525 }
6526
6527 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6528 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6529 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6530 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6531 VMMR0ThreadCtxHooksDeregister(pVCpu);
6532
6533 /* Leave HM context. This takes care of local init (term). */
6534 int rc = HMR0LeaveCpu(pVCpu);
6535
6536 HM_RESTORE_PREEMPT_IF_NEEDED();
6537
6538 return rc;
6539}
6540
6541
6542/**
6543 * Does the necessary state syncing before doing a longjmp to ring-3.
6544 *
6545 * @returns VBox status code.
6546 * @param pVM Pointer to the VM.
6547 * @param pVCpu Pointer to the VMCPU.
6548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6549 * out-of-sync. Make sure to update the required fields
6550 * before using them.
6551 *
6552 * @remarks No-long-jmp zone!!!
6553 */
6554DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6555{
6556 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6557}
6558
6559
6560/**
6561 * Take necessary actions before going back to ring-3.
6562 *
6563 * An action requires us to go back to ring-3. This function does the necessary
6564 * steps before we can safely return to ring-3. This is not the same as longjmps
6565 * to ring-3, this is voluntary and prepares the guest so it may continue
6566 * executing outside HM (recompiler/IEM).
6567 *
6568 * @returns VBox status code.
6569 * @param pVM Pointer to the VM.
6570 * @param pVCpu Pointer to the VMCPU.
6571 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6572 * out-of-sync. Make sure to update the required fields
6573 * before using them.
6574 * @param rcExit The reason for exiting to ring-3. Can be
6575 * VINF_VMM_UNKNOWN_RING3_CALL.
6576 */
6577static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6578{
6579 Assert(pVM);
6580 Assert(pVCpu);
6581 Assert(pMixedCtx);
6582 HMVMX_ASSERT_PREEMPT_SAFE();
6583
6584 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6585 {
6586 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6587 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6588 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6589 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6590 }
6591
6592 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6593 VMMRZCallRing3Disable(pVCpu);
6594 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6595
6596 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6597 if (pVCpu->hm.s.Event.fPending)
6598 {
6599 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6600 Assert(!pVCpu->hm.s.Event.fPending);
6601 }
6602
6603 /* Save guest state and restore host state bits. */
6604 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6605 AssertRCReturn(rc, rc);
6606 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6607
6608 /* Sync recompiler state. */
6609 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6610 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6611 | CPUM_CHANGED_LDTR
6612 | CPUM_CHANGED_GDTR
6613 | CPUM_CHANGED_IDTR
6614 | CPUM_CHANGED_TR
6615 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6616 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6617 if ( pVM->hm.s.fNestedPaging
6618 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6619 {
6620 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6621 }
6622
6623 Assert(!pVCpu->hm.s.fClearTrapFlag);
6624
6625 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6626 if (rcExit != VINF_EM_RAW_INTERRUPT)
6627 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6628
6629 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6630
6631 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6632 VMMRZCallRing3RemoveNotification(pVCpu);
6633 VMMRZCallRing3Enable(pVCpu);
6634
6635 return rc;
6636}
6637
6638
6639/**
6640 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6641 * longjump to ring-3 and possibly get preempted.
6642 *
6643 * @returns VBox status code.
6644 * @param pVCpu Pointer to the VMCPU.
6645 * @param enmOperation The operation causing the ring-3 longjump.
6646 * @param pvUser Opaque pointer to the guest-CPU context. The data
6647 * may be out-of-sync. Make sure to update the required
6648 * fields before using them.
6649 * @remarks If you modify code here, make sure to check whether
6650 * hmR0VmxLeave() needs to be updated too.
6651 */
6652DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6653{
6654 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6655 {
6656 VMMRZCallRing3RemoveNotification(pVCpu);
6657 HM_DISABLE_PREEMPT_IF_NEEDED();
6658
6659 /* If anything here asserts or fails, good luck. */
6660 if (CPUMIsGuestFPUStateActive(pVCpu))
6661 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6662
6663 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6664
6665#if HC_ARCH_BITS == 64
6666 /* Restore host-state bits that VT-x only restores partially. */
6667 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6668 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6669 {
6670 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6671 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6672 }
6673#endif
6674 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6675 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6676 {
6677 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6678 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6679 }
6680
6681 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6682 VMMR0ThreadCtxHooksDeregister(pVCpu);
6683
6684 HMR0LeaveCpu(pVCpu);
6685 HM_RESTORE_PREEMPT_IF_NEEDED();
6686 return VINF_SUCCESS;
6687 }
6688
6689 Assert(pVCpu);
6690 Assert(pvUser);
6691 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6692 HMVMX_ASSERT_PREEMPT_SAFE();
6693
6694 VMMRZCallRing3Disable(pVCpu);
6695 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6696
6697 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6698 enmOperation));
6699
6700 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6701 AssertRCReturn(rc, rc);
6702
6703 VMMRZCallRing3Enable(pVCpu);
6704 return VINF_SUCCESS;
6705}
6706
6707
6708/**
6709 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6710 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6711 *
6712 * @param pVCpu Pointer to the VMCPU.
6713 */
6714DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6715{
6716 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6717 {
6718 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6719 {
6720 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6721 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6722 AssertRC(rc);
6723 }
6724 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6725}
6726
6727
6728/**
6729 * Evaluates the event to be delivered to the guest and sets it as the pending
6730 * event.
6731 *
6732 * @param pVCpu Pointer to the VMCPU.
6733 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6734 * out-of-sync. Make sure to update the required fields
6735 * before using them.
6736 */
6737static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6738{
6739 Assert(!pVCpu->hm.s.Event.fPending);
6740
6741 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6742 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6743 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6744 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6745
6746 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6747 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6748 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6749 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6750 Assert(!TRPMHasTrap(pVCpu));
6751
6752 /** @todo SMI. SMIs take priority over NMIs. */
6753 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6754 {
6755 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6756 if ( !fBlockMovSS
6757 && !fBlockSti)
6758 {
6759 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6760 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6761 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6762 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6763
6764 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6765 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6766 }
6767 else
6768 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6769 }
6770 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6771 && !pVCpu->hm.s.fSingleInstruction)
6772 {
6773 /*
6774 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6775 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6776 * evaluated here and not set as pending, solely based on the force-flags.
6777 */
6778 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6779 AssertRC(rc);
6780 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6781 if ( !fBlockInt
6782 && !fBlockSti
6783 && !fBlockMovSS)
6784 {
6785 uint8_t u8Interrupt;
6786 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6787 if (RT_SUCCESS(rc))
6788 {
6789 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6790 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6791 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6792
6793 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6794 }
6795 else
6796 {
6797 /** @todo Does this actually happen? If not turn it into an assertion. */
6798 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6799 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6800 }
6801 }
6802 else
6803 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6804 }
6805}
6806
6807
6808/**
6809 * Injects any pending events into the guest if the guest is in a state to
6810 * receive them.
6811 *
6812 * @returns VBox status code (informational status codes included).
6813 * @param pVCpu Pointer to the VMCPU.
6814 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6815 * out-of-sync. Make sure to update the required fields
6816 * before using them.
6817 */
6818static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6819{
6820 HMVMX_ASSERT_PREEMPT_SAFE();
6821 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6822
6823 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6824 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6825 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6826 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6827
6828 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6829 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6830 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6831 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6832 Assert(!TRPMHasTrap(pVCpu));
6833
6834 int rc = VINF_SUCCESS;
6835 if (pVCpu->hm.s.Event.fPending)
6836 {
6837#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6838 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6839 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6840 {
6841 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6842 AssertRCReturn(rc, rc);
6843 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6844 Assert(!fBlockInt);
6845 Assert(!fBlockSti);
6846 Assert(!fBlockMovSS);
6847 }
6848 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6849 {
6850 Assert(!fBlockSti);
6851 Assert(!fBlockMovSS);
6852 }
6853#endif
6854 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6855 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6856 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6857 AssertRCReturn(rc, rc);
6858
6859 /* Update the interruptibility-state as it could have been changed by
6860 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6861 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6862 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6863
6864#ifdef VBOX_WITH_STATISTICS
6865 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6866 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6867 else
6868 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6869#endif
6870 }
6871
6872 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6873 int rc2 = VINF_SUCCESS;
6874 if ( fBlockSti
6875 || fBlockMovSS)
6876 {
6877 if ( !pVCpu->hm.s.fSingleInstruction
6878 && !DBGFIsStepping(pVCpu))
6879 {
6880 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6881 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6882 {
6883 /*
6884 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6885 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6886 * See Intel spec. 27.3.4 "Saving Non-Register State".
6887 */
6888 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6889 AssertRCReturn(rc, rc);
6890 }
6891 }
6892 else
6893 {
6894 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6895 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6896 uIntrState = 0;
6897 }
6898 }
6899
6900 /*
6901 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6902 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6903 */
6904 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6905 AssertRC(rc2);
6906
6907 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6908 return rc;
6909}
6910
6911
6912/**
6913 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6914 *
6915 * @param pVCpu Pointer to the VMCPU.
6916 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6917 * out-of-sync. Make sure to update the required fields
6918 * before using them.
6919 */
6920DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6921{
6922 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6923 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6924}
6925
6926
6927/**
6928 * Injects a double-fault (#DF) exception into the VM.
6929 *
6930 * @returns VBox status code (informational status code included).
6931 * @param pVCpu Pointer to the VMCPU.
6932 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6933 * out-of-sync. Make sure to update the required fields
6934 * before using them.
6935 */
6936DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6937{
6938 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6939 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6940 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6941 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6942 puIntrState);
6943}
6944
6945
6946/**
6947 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6948 *
6949 * @param pVCpu Pointer to the VMCPU.
6950 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6951 * out-of-sync. Make sure to update the required fields
6952 * before using them.
6953 */
6954DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6955{
6956 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6957 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6958 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6959}
6960
6961
6962/**
6963 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6964 *
6965 * @param pVCpu Pointer to the VMCPU.
6966 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6967 * out-of-sync. Make sure to update the required fields
6968 * before using them.
6969 * @param cbInstr The value of RIP that is to be pushed on the guest
6970 * stack.
6971 */
6972DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6973{
6974 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6975 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6976 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6977}
6978
6979
6980/**
6981 * Injects a general-protection (#GP) fault into the VM.
6982 *
6983 * @returns VBox status code (informational status code included).
6984 * @param pVCpu Pointer to the VMCPU.
6985 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6986 * out-of-sync. Make sure to update the required fields
6987 * before using them.
6988 * @param u32ErrorCode The error code associated with the #GP.
6989 */
6990DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6991 uint32_t *puIntrState)
6992{
6993 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6994 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6995 if (fErrorCodeValid)
6996 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6997 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6998 puIntrState);
6999}
7000
7001
7002/**
7003 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7004 *
7005 * @param pVCpu Pointer to the VMCPU.
7006 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7007 * out-of-sync. Make sure to update the required fields
7008 * before using them.
7009 * @param uVector The software interrupt vector number.
7010 * @param cbInstr The value of RIP that is to be pushed on the guest
7011 * stack.
7012 */
7013DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7014{
7015 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7016 if ( uVector == X86_XCPT_BP
7017 || uVector == X86_XCPT_OF)
7018 {
7019 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7020 }
7021 else
7022 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7023 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7024}
7025
7026
7027/**
7028 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7029 * stack.
7030 *
7031 * @returns VBox status code (information status code included).
7032 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7033 * @param pVM Pointer to the VM.
7034 * @param pMixedCtx Pointer to the guest-CPU context.
7035 * @param uValue The value to push to the guest stack.
7036 */
7037DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7038{
7039 /*
7040 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7041 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7042 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7043 */
7044 if (pMixedCtx->sp == 1)
7045 return VINF_EM_RESET;
7046 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7047 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7048 AssertRCReturn(rc, rc);
7049 return rc;
7050}
7051
7052
7053/**
7054 * Injects an event into the guest upon VM-entry by updating the relevant fields
7055 * in the VM-entry area in the VMCS.
7056 *
7057 * @returns VBox status code (informational error codes included).
7058 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7059 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7060 *
7061 * @param pVCpu Pointer to the VMCPU.
7062 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7063 * be out-of-sync. Make sure to update the required
7064 * fields before using them.
7065 * @param u64IntInfo The VM-entry interruption-information field.
7066 * @param cbInstr The VM-entry instruction length in bytes (for
7067 * software interrupts, exceptions and privileged
7068 * software exceptions).
7069 * @param u32ErrCode The VM-entry exception error code.
7070 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7071 * @param puIntrState Pointer to the current guest interruptibility-state.
7072 * This interruptibility-state will be updated if
7073 * necessary. This cannot not be NULL.
7074 *
7075 * @remarks Requires CR0!
7076 * @remarks No-long-jump zone!!!
7077 */
7078static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7079 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7080{
7081 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7082 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7083 Assert(puIntrState);
7084 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7085
7086 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7087 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7088
7089#ifdef VBOX_STRICT
7090 /* Validate the error-code-valid bit for hardware exceptions. */
7091 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7092 {
7093 switch (uVector)
7094 {
7095 case X86_XCPT_PF:
7096 case X86_XCPT_DF:
7097 case X86_XCPT_TS:
7098 case X86_XCPT_NP:
7099 case X86_XCPT_SS:
7100 case X86_XCPT_GP:
7101 case X86_XCPT_AC:
7102 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7103 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7104 /* fallthru */
7105 default:
7106 break;
7107 }
7108 }
7109#endif
7110
7111 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7112 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7113 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7114
7115 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7116
7117 /* We require CR0 to check if the guest is in real-mode. */
7118 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7119 AssertRCReturn(rc, rc);
7120
7121 /*
7122 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7123 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7124 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7125 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7126 */
7127 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7128 {
7129 PVM pVM = pVCpu->CTX_SUFF(pVM);
7130 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7131 {
7132 Assert(PDMVmmDevHeapIsEnabled(pVM));
7133 Assert(pVM->hm.s.vmx.pRealModeTSS);
7134
7135 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7136 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7137 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7138 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7139 AssertRCReturn(rc, rc);
7140 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7141
7142 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7143 const size_t cbIdtEntry = sizeof(X86IDTR16);
7144 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7145 {
7146 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7147 if (uVector == X86_XCPT_DF)
7148 return VINF_EM_RESET;
7149 else if (uVector == X86_XCPT_GP)
7150 {
7151 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7152 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7153 }
7154
7155 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7156 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7157 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7158 }
7159
7160 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7161 uint16_t uGuestIp = pMixedCtx->ip;
7162 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7163 {
7164 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7165 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7166 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7167 }
7168 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7169 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7170
7171 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7172 X86IDTR16 IdtEntry;
7173 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7174 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7175 AssertRCReturn(rc, rc);
7176
7177 /* Construct the stack frame for the interrupt/exception handler. */
7178 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7179 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7180 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7181 AssertRCReturn(rc, rc);
7182
7183 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7184 if (rc == VINF_SUCCESS)
7185 {
7186 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7187 pMixedCtx->rip = IdtEntry.offSel;
7188 pMixedCtx->cs.Sel = IdtEntry.uSel;
7189 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7190 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7191 && uVector == X86_XCPT_PF)
7192 {
7193 pMixedCtx->cr2 = GCPtrFaultAddress;
7194 }
7195
7196 /* If any other guest-state bits are changed here, make sure to update
7197 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7198 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7199 | HM_CHANGED_GUEST_RIP
7200 | HM_CHANGED_GUEST_RFLAGS
7201 | HM_CHANGED_GUEST_RSP);
7202
7203 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7204 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7205 {
7206 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7207 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7208 Log4(("Clearing inhibition due to STI.\n"));
7209 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7210 }
7211 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7212
7213 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7214 it, if we are returning to ring-3 before executing guest code. */
7215 pVCpu->hm.s.Event.fPending = false;
7216 }
7217 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7218 return rc;
7219 }
7220 else
7221 {
7222 /*
7223 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7224 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7225 */
7226 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7227 }
7228 }
7229
7230 /* Validate. */
7231 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7232 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7233 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7234
7235 /* Inject. */
7236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7237 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7238 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7239 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7240
7241 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7242 && uVector == X86_XCPT_PF)
7243 {
7244 pMixedCtx->cr2 = GCPtrFaultAddress;
7245 }
7246
7247 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7248 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7249
7250 AssertRCReturn(rc, rc);
7251 return rc;
7252}
7253
7254
7255/**
7256 * Clears the interrupt-window exiting control in the VMCS and if necessary
7257 * clears the current event in the VMCS as well.
7258 *
7259 * @returns VBox status code.
7260 * @param pVCpu Pointer to the VMCPU.
7261 *
7262 * @remarks Use this function only to clear events that have not yet been
7263 * delivered to the guest but are injected in the VMCS!
7264 * @remarks No-long-jump zone!!!
7265 */
7266static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7267{
7268 int rc;
7269 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7270
7271 /* Clear interrupt-window exiting control. */
7272 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7273 {
7274 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7275 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7276 AssertRC(rc);
7277 }
7278
7279 if (!pVCpu->hm.s.Event.fPending)
7280 return;
7281
7282#ifdef VBOX_STRICT
7283 uint32_t u32EntryInfo;
7284 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7285 AssertRC(rc);
7286 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7287#endif
7288
7289 /* Clear the entry-interruption field (including the valid bit). */
7290 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7291 AssertRC(rc);
7292
7293 /* Clear the pending debug exception field. */
7294 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7295 AssertRC(rc);
7296}
7297
7298
7299/**
7300 * Enters the VT-x session.
7301 *
7302 * @returns VBox status code.
7303 * @param pVM Pointer to the VM.
7304 * @param pVCpu Pointer to the VMCPU.
7305 * @param pCpu Pointer to the CPU info struct.
7306 */
7307VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7308{
7309 AssertPtr(pVM);
7310 AssertPtr(pVCpu);
7311 Assert(pVM->hm.s.vmx.fSupported);
7312 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7313 NOREF(pCpu);
7314
7315 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7316 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7317
7318#ifdef VBOX_STRICT
7319 /* Make sure we're in VMX root mode. */
7320 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7321 if (!(u32HostCR4 & X86_CR4_VMXE))
7322 {
7323 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7324 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7325 }
7326#endif
7327
7328 /*
7329 * Load the VCPU's VMCS as the current (and active) one.
7330 */
7331 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7332 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7333 if (RT_FAILURE(rc))
7334 return rc;
7335
7336 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7337 pVCpu->hm.s.fLeaveDone = false;
7338 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7339
7340 return VINF_SUCCESS;
7341}
7342
7343
7344/**
7345 * The thread-context callback (only on platforms which support it).
7346 *
7347 * @param enmEvent The thread-context event.
7348 * @param pVCpu Pointer to the VMCPU.
7349 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7350 * @thread EMT(pVCpu)
7351 */
7352VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7353{
7354 switch (enmEvent)
7355 {
7356 case RTTHREADCTXEVENT_PREEMPTING:
7357 {
7358 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7359 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7360 VMCPU_ASSERT_EMT(pVCpu);
7361
7362 PVM pVM = pVCpu->CTX_SUFF(pVM);
7363 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7364
7365 /* No longjmps (logger flushes, locks) in this fragile context. */
7366 VMMRZCallRing3Disable(pVCpu);
7367 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7368
7369 /*
7370 * Restore host-state (FPU, debug etc.)
7371 */
7372 if (!pVCpu->hm.s.fLeaveDone)
7373 {
7374 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7375 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7376 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7377 pVCpu->hm.s.fLeaveDone = true;
7378 }
7379
7380 /* Leave HM context, takes care of local init (term). */
7381 int rc = HMR0LeaveCpu(pVCpu);
7382 AssertRC(rc); NOREF(rc);
7383
7384 /* Restore longjmp state. */
7385 VMMRZCallRing3Enable(pVCpu);
7386 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7387 break;
7388 }
7389
7390 case RTTHREADCTXEVENT_RESUMED:
7391 {
7392 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7393 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7394 VMCPU_ASSERT_EMT(pVCpu);
7395
7396 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7397 VMMRZCallRing3Disable(pVCpu);
7398 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7399
7400 /* Initialize the bare minimum state required for HM. This takes care of
7401 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7402 int rc = HMR0EnterCpu(pVCpu);
7403 AssertRC(rc);
7404 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7405
7406 /* Load the active VMCS as the current one. */
7407 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7408 {
7409 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7410 AssertRC(rc); NOREF(rc);
7411 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7412 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7413 }
7414 pVCpu->hm.s.fLeaveDone = false;
7415
7416 /* Restore longjmp state. */
7417 VMMRZCallRing3Enable(pVCpu);
7418 break;
7419 }
7420
7421 default:
7422 break;
7423 }
7424}
7425
7426
7427/**
7428 * Saves the host state in the VMCS host-state.
7429 * Sets up the VM-exit MSR-load area.
7430 *
7431 * The CPU state will be loaded from these fields on every successful VM-exit.
7432 *
7433 * @returns VBox status code.
7434 * @param pVM Pointer to the VM.
7435 * @param pVCpu Pointer to the VMCPU.
7436 *
7437 * @remarks No-long-jump zone!!!
7438 */
7439static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7440{
7441 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7442
7443 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7444 return VINF_SUCCESS;
7445
7446 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7447 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7448
7449 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7450 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7451
7452 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7453 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7454
7455 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7456 return rc;
7457}
7458
7459
7460/**
7461 * Saves the host state in the VMCS host-state.
7462 *
7463 * @returns VBox status code.
7464 * @param pVM Pointer to the VM.
7465 * @param pVCpu Pointer to the VMCPU.
7466 *
7467 * @remarks No-long-jump zone!!!
7468 */
7469VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7470{
7471 AssertPtr(pVM);
7472 AssertPtr(pVCpu);
7473
7474 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7475
7476 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7477 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7478 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7479 return hmR0VmxSaveHostState(pVM, pVCpu);
7480}
7481
7482
7483/**
7484 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7485 * loaded from these fields on every successful VM-entry.
7486 *
7487 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7488 * Sets up the VM-entry controls.
7489 * Sets up the appropriate VMX non-root function to execute guest code based on
7490 * the guest CPU mode.
7491 *
7492 * @returns VBox status code.
7493 * @param pVM Pointer to the VM.
7494 * @param pVCpu Pointer to the VMCPU.
7495 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7496 * out-of-sync. Make sure to update the required fields
7497 * before using them.
7498 *
7499 * @remarks No-long-jump zone!!!
7500 */
7501static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7502{
7503 AssertPtr(pVM);
7504 AssertPtr(pVCpu);
7505 AssertPtr(pMixedCtx);
7506 HMVMX_ASSERT_PREEMPT_SAFE();
7507
7508#ifdef LOG_ENABLED
7509 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7510 * probably not initialized yet? Anyway this will do for now.
7511 *
7512 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7513 * interface and disable ring-3 calls when thread-context hooks are not
7514 * available. */
7515 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7516 VMMR0LogFlushDisable(pVCpu);
7517#endif
7518
7519 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7520
7521 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7522
7523 /* Determine real-on-v86 mode. */
7524 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7525 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7526 && CPUMIsGuestInRealModeEx(pMixedCtx))
7527 {
7528 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7529 }
7530
7531 /*
7532 * Load the guest-state into the VMCS.
7533 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7534 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7535 */
7536 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7537 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7538
7539 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7540 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7541 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7542
7543 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7544 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7545 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7546
7547 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7548 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7549
7550 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7551 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7552
7553 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7554 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7555 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7556
7557 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7558 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7559
7560 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7561 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7562
7563 /*
7564 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7565 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7566 */
7567 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7568 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7569
7570 /* Clear any unused and reserved bits. */
7571 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7572
7573#ifdef LOG_ENABLED
7574 /* Only reenable log-flushing if the caller has it enabled. */
7575 if (!fCallerDisabledLogFlush)
7576 VMMR0LogFlushEnable(pVCpu);
7577#endif
7578
7579 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7580 return rc;
7581}
7582
7583
7584/**
7585 * Loads the state shared between the host and guest into the VMCS.
7586 *
7587 * @param pVM Pointer to the VM.
7588 * @param pVCpu Pointer to the VMCPU.
7589 * @param pCtx Pointer to the guest-CPU context.
7590 *
7591 * @remarks No-long-jump zone!!!
7592 */
7593static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7594{
7595 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7596 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7597
7598 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7599 {
7600 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7601 AssertRC(rc);
7602 }
7603
7604 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7605 {
7606 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7607 AssertRC(rc);
7608
7609 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7610 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7611 {
7612 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7613 AssertRC(rc);
7614 }
7615 }
7616
7617 AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7618 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7619}
7620
7621
7622/**
7623 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7624 *
7625 * @param pVM Pointer to the VM.
7626 * @param pVCpu Pointer to the VMCPU.
7627 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7628 * out-of-sync. Make sure to update the required fields
7629 * before using them.
7630 */
7631DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7632{
7633 HMVMX_ASSERT_PREEMPT_SAFE();
7634
7635 Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7636#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7637 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7638#endif
7639
7640 if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7641 {
7642 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7643 AssertRC(rc);
7644 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7645 }
7646 else if (VMCPU_HMCF_VALUE(pVCpu))
7647 {
7648 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7649 AssertRC(rc);
7650 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7651 }
7652
7653 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7654 AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7655 || VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
7656 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7657
7658#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7659 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7660 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7661 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7662#endif
7663}
7664
7665
7666/**
7667 * Does the preparations before executing guest code in VT-x.
7668 *
7669 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7670 * recompiler. We must be cautious what we do here regarding committing
7671 * guest-state information into the VMCS assuming we assuredly execute the
7672 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7673 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7674 * so that the recompiler can (and should) use them when it resumes guest
7675 * execution. Otherwise such operations must be done when we can no longer
7676 * exit to ring-3.
7677 *
7678 * @returns Strict VBox status code.
7679 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7680 * have been disabled.
7681 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7682 * double-fault into the guest.
7683 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7684 *
7685 * @param pVM Pointer to the VM.
7686 * @param pVCpu Pointer to the VMCPU.
7687 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7688 * out-of-sync. Make sure to update the required fields
7689 * before using them.
7690 * @param pVmxTransient Pointer to the VMX transient structure.
7691 *
7692 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7693 * interrupts will be disabled.
7694 */
7695static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7696{
7697 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7698
7699#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7700 PGMRZDynMapFlushAutoSet(pVCpu);
7701#endif
7702
7703 /* Check force flag actions that might require us to go back to ring-3. */
7704 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7705 if (rc != VINF_SUCCESS)
7706 return rc;
7707
7708#ifndef IEM_VERIFICATION_MODE_FULL
7709 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7710 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7711 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7712 {
7713 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7714 RTGCPHYS GCPhysApicBase;
7715 GCPhysApicBase = pMixedCtx->msrApicBase;
7716 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7717
7718 /* Unalias any existing mapping. */
7719 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7720 AssertRCReturn(rc, rc);
7721
7722 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7723 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7724 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7725 AssertRCReturn(rc, rc);
7726
7727 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7728 }
7729#endif /* !IEM_VERIFICATION_MODE_FULL */
7730
7731 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7732 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7733
7734 /*
7735 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7736 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7737 */
7738 if (TRPMHasTrap(pVCpu))
7739 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7740 else if (!pVCpu->hm.s.Event.fPending)
7741 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7742
7743 /*
7744 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7745 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7746 */
7747 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7748 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7749 {
7750 Assert(rc == VINF_EM_RESET);
7751 return rc;
7752 }
7753
7754 /*
7755 * No longjmps to ring-3 from this point on!!!
7756 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7757 * This also disables flushing of the R0-logger instance (if any).
7758 */
7759 VMMRZCallRing3Disable(pVCpu);
7760
7761 /*
7762 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7763 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7764 *
7765 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7766 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7767 *
7768 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7769 * executing guest code.
7770 */
7771 pVmxTransient->uEflags = ASMIntDisableFlags();
7772 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7773 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7774 {
7775 hmR0VmxClearEventVmcs(pVCpu);
7776 ASMSetFlags(pVmxTransient->uEflags);
7777 VMMRZCallRing3Enable(pVCpu);
7778 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7779 return VINF_EM_RAW_TO_R3;
7780 }
7781 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7782 {
7783 hmR0VmxClearEventVmcs(pVCpu);
7784 ASMSetFlags(pVmxTransient->uEflags);
7785 VMMRZCallRing3Enable(pVCpu);
7786 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7787 return VINF_EM_RAW_INTERRUPT;
7788 }
7789
7790 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7791 pVCpu->hm.s.Event.fPending = false;
7792
7793 return VINF_SUCCESS;
7794}
7795
7796
7797/**
7798 * Prepares to run guest code in VT-x and we've committed to doing so. This
7799 * means there is no backing out to ring-3 or anywhere else at this
7800 * point.
7801 *
7802 * @param pVM Pointer to the VM.
7803 * @param pVCpu Pointer to the VMCPU.
7804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7805 * out-of-sync. Make sure to update the required fields
7806 * before using them.
7807 * @param pVmxTransient Pointer to the VMX transient structure.
7808 *
7809 * @remarks Called with preemption disabled.
7810 * @remarks No-long-jump zone!!!
7811 */
7812static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7813{
7814 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7815 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7816 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7817
7818 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7819 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7820
7821 /*
7822 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7823 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7824 * Reload only the necessary state, the assertion will catch if other parts of the code
7825 * change.
7826 */
7827 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7828 {
7829 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7830 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7831 }
7832
7833#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7834 if (!CPUMIsGuestFPUStateActive(pVCpu))
7835 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7836 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7837#endif
7838
7839 if ( pVCpu->hm.s.fUseGuestFpu
7840 && !CPUMIsGuestFPUStateActive(pVCpu))
7841 {
7842 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7843 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
7844 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7845 }
7846
7847 /*
7848 * Load the host state bits as we may've been preempted (only happens when
7849 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7850 */
7851 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7852 {
7853 /* This ASSUMES that pfnStartVM has been set up already. */
7854 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7855 AssertRC(rc);
7856 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7857 }
7858 Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7859
7860 /*
7861 * Load the state shared between host and guest (FPU, debug).
7862 */
7863 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7864 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7865 AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7866
7867 /* Store status of the shared guest-host state at the time of VM-entry. */
7868#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7869 if (CPUMIsGuestInLongModeEx(pMixedCtx))
7870 {
7871 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7872 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7873 }
7874 else
7875#endif
7876 {
7877 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7878 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7879 }
7880 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7881
7882 /*
7883 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7884 */
7885 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7886 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7887
7888 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7889 RTCPUID idCurrentCpu = pCpu->idCpu;
7890 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7891 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7892 {
7893 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7894 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7895 }
7896
7897 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7898 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7899 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7900 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7901
7902 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7903
7904 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7905 to start executing. */
7906
7907#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7908 /*
7909 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7910 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7911 */
7912 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7913 {
7914 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7915 uint64_t u64HostTscAux = 0;
7916 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7917 AssertRC(rc2);
7918 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7919 }
7920#endif
7921}
7922
7923
7924/**
7925 * Performs some essential restoration of state after running guest code in
7926 * VT-x.
7927 *
7928 * @param pVM Pointer to the VM.
7929 * @param pVCpu Pointer to the VMCPU.
7930 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7931 * out-of-sync. Make sure to update the required fields
7932 * before using them.
7933 * @param pVmxTransient Pointer to the VMX transient structure.
7934 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7935 *
7936 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7937 *
7938 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7939 * unconditionally when it is safe to do so.
7940 */
7941static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7942{
7943 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7944
7945 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7946 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7947 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7948 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7949 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7950
7951 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7952 {
7953#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7954 /* Restore host's TSC_AUX. */
7955 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7956 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7957#endif
7958 /** @todo Find a way to fix hardcoding a guestimate. */
7959 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7960 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7961 }
7962
7963 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7964 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7965 Assert(!(ASMGetFlags() & X86_EFL_IF));
7966 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7967
7968#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7969 if (CPUMIsGuestFPUStateActive(pVCpu))
7970 {
7971 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7972 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7973 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7974 }
7975#endif
7976
7977 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7978 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7979 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7980 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7981
7982 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7983 uint32_t uExitReason;
7984 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7985 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7986 AssertRC(rc);
7987 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7988 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
7989
7990 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7991 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7992 {
7993 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7994 pVmxTransient->fVMEntryFailed));
7995 return;
7996 }
7997
7998 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7999 {
8000 /* Update the guest interruptibility-state from the VMCS. */
8001 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8002#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8003 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8004 AssertRC(rc);
8005#endif
8006 /*
8007 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8008 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8009 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8010 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8011 */
8012 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8013 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8014 {
8015 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8016 AssertRC(rc);
8017 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8018 }
8019 }
8020}
8021
8022
8023
8024/**
8025 * Runs the guest code using VT-x the normal way.
8026 *
8027 * @returns VBox status code.
8028 * @param pVM Pointer to the VM.
8029 * @param pVCpu Pointer to the VMCPU.
8030 * @param pCtx Pointer to the guest-CPU context.
8031 *
8032 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
8033 * @remarks Called with preemption disabled.
8034 */
8035static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8036{
8037 VMXTRANSIENT VmxTransient;
8038 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8039 int rc = VERR_INTERNAL_ERROR_5;
8040 uint32_t cLoops = 0;
8041
8042 for (;; cLoops++)
8043 {
8044 Assert(!HMR0SuspendPending());
8045 HMVMX_ASSERT_CPU_SAFE();
8046
8047 /* Preparatory work for running guest code, this may force us to return
8048 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8049 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8050 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8051 if (rc != VINF_SUCCESS)
8052 break;
8053
8054 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8055 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8056 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8057
8058 /* Restore any residual host-state and save any bits shared between host
8059 and guest into the guest-CPU state. Re-enables interrupts! */
8060 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8061
8062 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8063 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8064 {
8065 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8066 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8067 return rc;
8068 }
8069
8070 /* Handle the VM-exit. */
8071 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8072 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8073 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8074 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8075 HMVMX_START_EXIT_DISPATCH_PROF();
8076#ifdef HMVMX_USE_FUNCTION_TABLE
8077 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8078#else
8079 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8080#endif
8081 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8082 if (rc != VINF_SUCCESS)
8083 break;
8084 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8085 {
8086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8087 rc = VINF_EM_RAW_INTERRUPT;
8088 break;
8089 }
8090 }
8091
8092 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8093 return rc;
8094}
8095
8096
8097/**
8098 * Single steps guest code using VT-x.
8099 *
8100 * @returns VBox status code.
8101 * @param pVM Pointer to the VM.
8102 * @param pVCpu Pointer to the VMCPU.
8103 * @param pCtx Pointer to the guest-CPU context.
8104 *
8105 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8106 * @remarks Called with preemption disabled.
8107 */
8108static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8109{
8110 VMXTRANSIENT VmxTransient;
8111 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8112 int rc = VERR_INTERNAL_ERROR_5;
8113 uint32_t cLoops = 0;
8114 uint16_t uCsStart = pCtx->cs.Sel;
8115 uint64_t uRipStart = pCtx->rip;
8116
8117 for (;; cLoops++)
8118 {
8119 Assert(!HMR0SuspendPending());
8120 HMVMX_ASSERT_CPU_SAFE();
8121
8122 /* Preparatory work for running guest code, this may force us to return
8123 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8124 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8125 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8126 if (rc != VINF_SUCCESS)
8127 break;
8128
8129 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8130 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8131 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8132
8133 /* Restore any residual host-state and save any bits shared between host
8134 and guest into the guest-CPU state. Re-enables interrupts! */
8135 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8136
8137 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8138 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8139 {
8140 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8141 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8142 return rc;
8143 }
8144
8145 /* Handle the VM-exit. */
8146 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8147 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8148 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8149 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8150 HMVMX_START_EXIT_DISPATCH_PROF();
8151#ifdef HMVMX_USE_FUNCTION_TABLE
8152 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8153#else
8154 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8155#endif
8156 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8157 if (rc != VINF_SUCCESS)
8158 break;
8159 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8160 {
8161 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8162 rc = VINF_EM_RAW_INTERRUPT;
8163 break;
8164 }
8165
8166 /*
8167 * Did the RIP change, if so, consider it a single step.
8168 * Otherwise, make sure one of the TFs gets set.
8169 */
8170 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8171 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8172 AssertRCReturn(rc2, rc2);
8173 if ( pCtx->rip != uRipStart
8174 || pCtx->cs.Sel != uCsStart)
8175 {
8176 rc = VINF_EM_DBG_STEPPED;
8177 break;
8178 }
8179 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8180 }
8181
8182 /*
8183 * Clear the X86_EFL_TF if necessary.
8184 */
8185 if (pVCpu->hm.s.fClearTrapFlag)
8186 {
8187 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8188 AssertRCReturn(rc2, rc2);
8189 pVCpu->hm.s.fClearTrapFlag = false;
8190 pCtx->eflags.Bits.u1TF = 0;
8191 }
8192 /** @todo there seems to be issues with the resume flag when the monitor trap
8193 * flag is pending without being used. Seen early in bios init when
8194 * accessing APIC page in prot mode. */
8195
8196 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8197 return rc;
8198}
8199
8200
8201/**
8202 * Runs the guest code using VT-x.
8203 *
8204 * @returns VBox status code.
8205 * @param pVM Pointer to the VM.
8206 * @param pVCpu Pointer to the VMCPU.
8207 * @param pCtx Pointer to the guest-CPU context.
8208 *
8209 * @remarks Called with preemption disabled.
8210 */
8211VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8212{
8213 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8214 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8215 HMVMX_ASSERT_PREEMPT_SAFE();
8216
8217 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8218
8219 int rc;
8220 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8221 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8222 else
8223 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8224
8225 if (rc == VERR_EM_INTERPRETER)
8226 rc = VINF_EM_RAW_EMULATE_INSTR;
8227 else if (rc == VINF_EM_RESET)
8228 rc = VINF_EM_TRIPLE_FAULT;
8229
8230 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8231 if (RT_FAILURE(rc2))
8232 {
8233 pVCpu->hm.s.u32HMError = rc;
8234 rc = rc2;
8235 }
8236 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8237 return rc;
8238}
8239
8240
8241#ifndef HMVMX_USE_FUNCTION_TABLE
8242DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8243{
8244 int rc;
8245 switch (rcReason)
8246 {
8247 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8248 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8249 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8250 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8251 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8252 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8253 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8254 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8255 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8256 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8257 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8258 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8259 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8260 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8261 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8262 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8263 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8264 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8265 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8266 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8267 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8268 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8269 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8270 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8271 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8272 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8273 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8274 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8275 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8276 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8277 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8278 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8279 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8280
8281 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8282 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8283 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8284 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8285 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8286 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8287 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8288 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8289 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8290
8291 case VMX_EXIT_VMCALL:
8292 case VMX_EXIT_VMCLEAR:
8293 case VMX_EXIT_VMLAUNCH:
8294 case VMX_EXIT_VMPTRLD:
8295 case VMX_EXIT_VMPTRST:
8296 case VMX_EXIT_VMREAD:
8297 case VMX_EXIT_VMRESUME:
8298 case VMX_EXIT_VMWRITE:
8299 case VMX_EXIT_VMXOFF:
8300 case VMX_EXIT_VMXON:
8301 case VMX_EXIT_INVEPT:
8302 case VMX_EXIT_INVVPID:
8303 case VMX_EXIT_VMFUNC:
8304 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8305 break;
8306 default:
8307 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8308 break;
8309 }
8310 return rc;
8311}
8312#endif
8313
8314#ifdef DEBUG
8315/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8316# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8317 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8318
8319# define HMVMX_ASSERT_PREEMPT_CPUID() \
8320 do \
8321 { \
8322 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8323 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8324 } while (0)
8325
8326# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8327 do { \
8328 AssertPtr(pVCpu); \
8329 AssertPtr(pMixedCtx); \
8330 AssertPtr(pVmxTransient); \
8331 Assert(pVmxTransient->fVMEntryFailed == false); \
8332 Assert(ASMIntAreEnabled()); \
8333 HMVMX_ASSERT_PREEMPT_SAFE(); \
8334 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8335 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8336 HMVMX_ASSERT_PREEMPT_SAFE(); \
8337 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8338 HMVMX_ASSERT_PREEMPT_CPUID(); \
8339 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8340 } while (0)
8341
8342# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8343 do { \
8344 Log4Func(("\n")); \
8345 } while(0)
8346#else /* Release builds */
8347# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8348# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8349#endif
8350
8351
8352/**
8353 * Advances the guest RIP after reading it from the VMCS.
8354 *
8355 * @returns VBox status code.
8356 * @param pVCpu Pointer to the VMCPU.
8357 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8358 * out-of-sync. Make sure to update the required fields
8359 * before using them.
8360 * @param pVmxTransient Pointer to the VMX transient structure.
8361 *
8362 * @remarks No-long-jump zone!!!
8363 */
8364DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8365{
8366 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8367 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8368 AssertRCReturn(rc, rc);
8369
8370 pMixedCtx->rip += pVmxTransient->cbInstr;
8371 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8372 return rc;
8373}
8374
8375
8376/**
8377 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8378 * and update error record fields accordingly.
8379 *
8380 * @return VMX_IGS_* return codes.
8381 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8382 * wrong with the guest state.
8383 *
8384 * @param pVM Pointer to the VM.
8385 * @param pVCpu Pointer to the VMCPU.
8386 * @param pCtx Pointer to the guest-CPU state.
8387 */
8388static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8389{
8390#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8391#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8392 uError = (err); \
8393 break; \
8394 } else do {} while (0)
8395/* Duplicate of IEM_IS_CANONICAL(). */
8396#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8397
8398 int rc;
8399 uint32_t uError = VMX_IGS_ERROR;
8400 uint32_t u32Val;
8401 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8402
8403 do
8404 {
8405 /*
8406 * CR0.
8407 */
8408 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8409 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8410 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8411 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8412 if (fUnrestrictedGuest)
8413 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8414
8415 uint32_t u32GuestCR0;
8416 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8417 AssertRCBreak(rc);
8418 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8419 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8420 if ( !fUnrestrictedGuest
8421 && (u32GuestCR0 & X86_CR0_PG)
8422 && !(u32GuestCR0 & X86_CR0_PE))
8423 {
8424 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8425 }
8426
8427 /*
8428 * CR4.
8429 */
8430 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8431 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8432
8433 uint32_t u32GuestCR4;
8434 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8435 AssertRCBreak(rc);
8436 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8437 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8438
8439 /*
8440 * IA32_DEBUGCTL MSR.
8441 */
8442 uint64_t u64Val;
8443 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8444 AssertRCBreak(rc);
8445 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8446 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8447 {
8448 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8449 }
8450 uint64_t u64DebugCtlMsr = u64Val;
8451
8452#ifdef VBOX_STRICT
8453 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8454 AssertRCBreak(rc);
8455 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8456#endif
8457 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8458
8459 /*
8460 * RIP and RFLAGS.
8461 */
8462 uint32_t u32Eflags;
8463#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8464 if (HMVMX_IS_64BIT_HOST_MODE())
8465 {
8466 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8467 AssertRCBreak(rc);
8468 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8469 if ( !fLongModeGuest
8470 || !pCtx->cs.Attr.n.u1Long)
8471 {
8472 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8473 }
8474 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8475 * must be identical if the "IA32e mode guest" VM-entry control is 1
8476 * and CS.L is 1. No check applies if the CPU supports 64
8477 * linear-address bits. */
8478
8479 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8480 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8481 AssertRCBreak(rc);
8482 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8483 VMX_IGS_RFLAGS_RESERVED);
8484 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8485 u32Eflags = u64Val;
8486 }
8487 else
8488#endif
8489 {
8490 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8491 AssertRCBreak(rc);
8492 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8493 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8494 }
8495
8496 if ( fLongModeGuest
8497 || ( fUnrestrictedGuest
8498 && !(u32GuestCR0 & X86_CR0_PE)))
8499 {
8500 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8501 }
8502
8503 uint32_t u32EntryInfo;
8504 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8505 AssertRCBreak(rc);
8506 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8507 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8508 {
8509 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8510 }
8511
8512 /*
8513 * 64-bit checks.
8514 */
8515#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8516 if (HMVMX_IS_64BIT_HOST_MODE())
8517 {
8518 if ( fLongModeGuest
8519 && !fUnrestrictedGuest)
8520 {
8521 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8522 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8523 }
8524
8525 if ( !fLongModeGuest
8526 && (u32GuestCR4 & X86_CR4_PCIDE))
8527 {
8528 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8529 }
8530
8531 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8532 * 51:32 beyond the processor's physical-address width are 0. */
8533
8534 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8535 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8536 {
8537 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8538 }
8539
8540 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8541 AssertRCBreak(rc);
8542 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8543
8544 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8545 AssertRCBreak(rc);
8546 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8547 }
8548#endif
8549
8550 /*
8551 * PERF_GLOBAL MSR.
8552 */
8553 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8554 {
8555 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8556 AssertRCBreak(rc);
8557 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8558 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8559 }
8560
8561 /*
8562 * PAT MSR.
8563 */
8564 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8565 {
8566 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8567 AssertRCBreak(rc);
8568 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8569 for (unsigned i = 0; i < 8; i++)
8570 {
8571 uint8_t u8Val = (u64Val & 0x7);
8572 if ( u8Val != 0 /* UC */
8573 || u8Val != 1 /* WC */
8574 || u8Val != 4 /* WT */
8575 || u8Val != 5 /* WP */
8576 || u8Val != 6 /* WB */
8577 || u8Val != 7 /* UC- */)
8578 {
8579 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8580 }
8581 u64Val >>= 3;
8582 }
8583 }
8584
8585 /*
8586 * EFER MSR.
8587 */
8588 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8589 {
8590 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8591 AssertRCBreak(rc);
8592 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8593 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8594 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8595 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8596 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8597 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8598 }
8599
8600 /*
8601 * Segment registers.
8602 */
8603 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8604 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8605 if (!(u32Eflags & X86_EFL_VM))
8606 {
8607 /* CS */
8608 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8609 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8610 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8611 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8612 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8613 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8614 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8615 /* CS cannot be loaded with NULL in protected mode. */
8616 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8617 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8618 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8619 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8620 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8621 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8622 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8623 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8624 else
8625 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8626
8627 /* SS */
8628 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8629 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8630 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8631 if ( !(pCtx->cr0 & X86_CR0_PE)
8632 || pCtx->cs.Attr.n.u4Type == 3)
8633 {
8634 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8635 }
8636 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8637 {
8638 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8639 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8640 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8641 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8642 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8643 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8644 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8645 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8646 }
8647
8648 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8649 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8650 {
8651 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8652 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8653 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8654 || pCtx->ds.Attr.n.u4Type > 11
8655 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8656 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8657 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8658 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8659 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8660 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8661 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8662 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8663 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8664 }
8665 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8666 {
8667 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8668 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8669 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8670 || pCtx->es.Attr.n.u4Type > 11
8671 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8672 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8673 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8674 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8675 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8676 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8677 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8678 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8679 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8680 }
8681 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8682 {
8683 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8684 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8685 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8686 || pCtx->fs.Attr.n.u4Type > 11
8687 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8688 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8689 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8690 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8691 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8692 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8693 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8694 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8695 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8696 }
8697 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8698 {
8699 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8700 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8701 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8702 || pCtx->gs.Attr.n.u4Type > 11
8703 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8704 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8705 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8706 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8707 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8708 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8709 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8710 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8711 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8712 }
8713 /* 64-bit capable CPUs. */
8714#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8715 if (HMVMX_IS_64BIT_HOST_MODE())
8716 {
8717 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8718 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8719 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8720 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8721 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8722 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8723 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8724 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8725 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8726 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8727 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8728 }
8729#endif
8730 }
8731 else
8732 {
8733 /* V86 mode checks. */
8734 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8735 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8736 {
8737 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8738 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8739 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8740 }
8741 else
8742 {
8743 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8744 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8745 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8746 }
8747
8748 /* CS */
8749 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8750 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8751 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8752 /* SS */
8753 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8754 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8755 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8756 /* DS */
8757 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8758 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8759 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8760 /* ES */
8761 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8762 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8763 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8764 /* FS */
8765 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8766 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8767 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8768 /* GS */
8769 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8770 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8771 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8772 /* 64-bit capable CPUs. */
8773#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8774 if (HMVMX_IS_64BIT_HOST_MODE())
8775 {
8776 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8777 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8778 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8779 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8780 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8781 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8782 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8783 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8784 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8785 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8786 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8787 }
8788#endif
8789 }
8790
8791 /*
8792 * TR.
8793 */
8794 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8795 /* 64-bit capable CPUs. */
8796#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8797 if (HMVMX_IS_64BIT_HOST_MODE())
8798 {
8799 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8800 }
8801#endif
8802 if (fLongModeGuest)
8803 {
8804 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8805 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8806 }
8807 else
8808 {
8809 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8810 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8811 VMX_IGS_TR_ATTR_TYPE_INVALID);
8812 }
8813 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8814 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8815 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8816 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8817 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8818 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8819 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8820 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8821
8822 /*
8823 * GDTR and IDTR.
8824 */
8825#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8826 if (HMVMX_IS_64BIT_HOST_MODE())
8827 {
8828 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8829 AssertRCBreak(rc);
8830 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8831
8832 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8833 AssertRCBreak(rc);
8834 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8835 }
8836#endif
8837
8838 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8839 AssertRCBreak(rc);
8840 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8841
8842 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8843 AssertRCBreak(rc);
8844 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8845
8846 /*
8847 * Guest Non-Register State.
8848 */
8849 /* Activity State. */
8850 uint32_t u32ActivityState;
8851 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8852 AssertRCBreak(rc);
8853 HMVMX_CHECK_BREAK( !u32ActivityState
8854 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8855 VMX_IGS_ACTIVITY_STATE_INVALID);
8856 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8857 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8858 uint32_t u32IntrState;
8859 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8860 AssertRCBreak(rc);
8861 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8862 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8863 {
8864 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8865 }
8866
8867 /** @todo Activity state and injecting interrupts. Left as a todo since we
8868 * currently don't use activity states but ACTIVE. */
8869
8870 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8871 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8872
8873 /* Guest interruptibility-state. */
8874 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8875 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8876 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8877 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8878 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8879 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8880 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8881 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8882 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8883 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
8884 {
8885 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8886 {
8887 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8888 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8889 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8890 }
8891 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8892 {
8893 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8894 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8895 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8896 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8897 }
8898 }
8899 /** @todo Assumes the processor is not in SMM. */
8900 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8901 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8902 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8903 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8904 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8905 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8906 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8907 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8908 {
8909 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8910 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8911 }
8912
8913 /* Pending debug exceptions. */
8914 if (HMVMX_IS_64BIT_HOST_MODE())
8915 {
8916 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8917 AssertRCBreak(rc);
8918 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8919 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8920 u32Val = u64Val; /* For pending debug exceptions checks below. */
8921 }
8922 else
8923 {
8924 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8925 AssertRCBreak(rc);
8926 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8927 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8928 }
8929
8930 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8931 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8932 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8933 {
8934 if ( (u32Eflags & X86_EFL_TF)
8935 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8936 {
8937 /* Bit 14 is PendingDebug.BS. */
8938 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8939 }
8940 if ( !(u32Eflags & X86_EFL_TF)
8941 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8942 {
8943 /* Bit 14 is PendingDebug.BS. */
8944 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8945 }
8946 }
8947
8948 /* VMCS link pointer. */
8949 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8950 AssertRCBreak(rc);
8951 if (u64Val != UINT64_C(0xffffffffffffffff))
8952 {
8953 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8954 /** @todo Bits beyond the processor's physical-address width MBZ. */
8955 /** @todo 32-bit located in memory referenced by value of this field (as a
8956 * physical address) must contain the processor's VMCS revision ID. */
8957 /** @todo SMM checks. */
8958 }
8959
8960 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8961
8962 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8963 if (uError == VMX_IGS_ERROR)
8964 uError = VMX_IGS_REASON_NOT_FOUND;
8965 } while (0);
8966
8967 pVCpu->hm.s.u32HMError = uError;
8968 return uError;
8969
8970#undef HMVMX_ERROR_BREAK
8971#undef HMVMX_CHECK_BREAK
8972#undef HMVMX_IS_CANONICAL
8973}
8974
8975/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8976/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8977/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8978
8979/** @name VM-exit handlers.
8980 * @{
8981 */
8982
8983/**
8984 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8985 */
8986HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8987{
8988 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8990 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8991#if HC_ARCH_BITS == 64
8992 Assert(ASMIntAreEnabled());
8993 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8994 return VINF_SUCCESS;
8995#endif
8996 return VINF_EM_RAW_INTERRUPT;
8997}
8998
8999
9000/**
9001 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9002 */
9003HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9004{
9005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9006 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9007
9008 int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
9009 AssertRCReturn(rc, rc);
9010
9011 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9012 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9013 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9014 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9015
9016 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9017 {
9018 /*
9019 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9020 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9021 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9022 *
9023 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9024 */
9025 VMXDispatchHostNmi();
9026 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9027 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9028 return VINF_SUCCESS;
9029 }
9030
9031 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9032 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9033 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9034 {
9035 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9036 return VINF_SUCCESS;
9037 }
9038 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9039 {
9040 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9041 return rc;
9042 }
9043
9044 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9045 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9046 switch (uIntType)
9047 {
9048 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9049 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
9050 /* no break */
9051 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9052 {
9053 switch (uVector)
9054 {
9055 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9056 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9057 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9058 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9059 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9060 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9061#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9062 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9063 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9064 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9065 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9066 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9067 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9068 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9069 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9070 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9071 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9072#endif
9073 default:
9074 {
9075 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9076 AssertRCReturn(rc, rc);
9077
9078 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9079 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9080 {
9081 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9082 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9083 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9084
9085 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9086 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
9087 AssertRCReturn(rc, rc);
9088 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9089 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9090 0 /* GCPtrFaultAddress */);
9091 AssertRCReturn(rc, rc);
9092 }
9093 else
9094 {
9095 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9096 pVCpu->hm.s.u32HMError = uVector;
9097 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9098 }
9099 break;
9100 }
9101 }
9102 break;
9103 }
9104
9105 default:
9106 {
9107 pVCpu->hm.s.u32HMError = uExitIntInfo;
9108 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9109 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9110 break;
9111 }
9112 }
9113 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9114 return rc;
9115}
9116
9117
9118/**
9119 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9120 */
9121HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9122{
9123 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9124
9125 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9126 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9127 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9128 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9129 AssertRCReturn(rc, rc);
9130
9131 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9132 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9133 return VINF_SUCCESS;
9134}
9135
9136
9137/**
9138 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9139 */
9140HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9141{
9142 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9143 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9144 HMVMX_RETURN_UNEXPECTED_EXIT();
9145}
9146
9147
9148/**
9149 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9150 */
9151HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9152{
9153 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9155 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9156}
9157
9158
9159/**
9160 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9161 */
9162HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9163{
9164 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9165 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9166 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9167}
9168
9169
9170/**
9171 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9172 */
9173HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9174{
9175 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9176 PVM pVM = pVCpu->CTX_SUFF(pVM);
9177 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9178 if (RT_LIKELY(rc == VINF_SUCCESS))
9179 {
9180 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9181 Assert(pVmxTransient->cbInstr == 2);
9182 }
9183 else
9184 {
9185 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9186 rc = VERR_EM_INTERPRETER;
9187 }
9188 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9189 return rc;
9190}
9191
9192
9193/**
9194 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9195 */
9196HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9197{
9198 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9199 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9200 AssertRCReturn(rc, rc);
9201
9202 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9203 return VINF_EM_RAW_EMULATE_INSTR;
9204
9205 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9206 HMVMX_RETURN_UNEXPECTED_EXIT();
9207}
9208
9209
9210/**
9211 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9212 */
9213HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9214{
9215 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9216 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9217 AssertRCReturn(rc, rc);
9218
9219 PVM pVM = pVCpu->CTX_SUFF(pVM);
9220 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9221 if (RT_LIKELY(rc == VINF_SUCCESS))
9222 {
9223 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9224 Assert(pVmxTransient->cbInstr == 2);
9225 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9226 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9227 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9228 }
9229 else
9230 {
9231 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9232 rc = VERR_EM_INTERPRETER;
9233 }
9234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9235 return rc;
9236}
9237
9238
9239/**
9240 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9241 */
9242HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9243{
9244 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9245 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9246 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9247 AssertRCReturn(rc, rc);
9248
9249 PVM pVM = pVCpu->CTX_SUFF(pVM);
9250 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9251 if (RT_LIKELY(rc == VINF_SUCCESS))
9252 {
9253 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9254 Assert(pVmxTransient->cbInstr == 3);
9255 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9256 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9257 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9258 }
9259 else
9260 {
9261 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9262 rc = VERR_EM_INTERPRETER;
9263 }
9264 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9265 return rc;
9266}
9267
9268
9269/**
9270 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9271 */
9272HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9273{
9274 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9275 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9276 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9277 AssertRCReturn(rc, rc);
9278
9279 PVM pVM = pVCpu->CTX_SUFF(pVM);
9280 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9281 if (RT_LIKELY(rc == VINF_SUCCESS))
9282 {
9283 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9284 Assert(pVmxTransient->cbInstr == 2);
9285 }
9286 else
9287 {
9288 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9289 rc = VERR_EM_INTERPRETER;
9290 }
9291 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9292 return rc;
9293}
9294
9295
9296/**
9297 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9298 */
9299HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9300{
9301 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9302 PVM pVM = pVCpu->CTX_SUFF(pVM);
9303 Assert(!pVM->hm.s.fNestedPaging);
9304
9305 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9306 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9307 AssertRCReturn(rc, rc);
9308
9309 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9310 rc = VBOXSTRICTRC_VAL(rc2);
9311 if (RT_LIKELY(rc == VINF_SUCCESS))
9312 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9313 else
9314 {
9315 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9316 pVmxTransient->uExitQualification, rc));
9317 }
9318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9319 return rc;
9320}
9321
9322
9323/**
9324 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9325 */
9326HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9327{
9328 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9329 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9330 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9331 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9332 AssertRCReturn(rc, rc);
9333
9334 PVM pVM = pVCpu->CTX_SUFF(pVM);
9335 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9336 if (RT_LIKELY(rc == VINF_SUCCESS))
9337 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9338 else
9339 {
9340 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9341 rc = VERR_EM_INTERPRETER;
9342 }
9343 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9344 return rc;
9345}
9346
9347
9348/**
9349 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9350 */
9351HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9352{
9353 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9354 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9355 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9356 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9357 AssertRCReturn(rc, rc);
9358
9359 PVM pVM = pVCpu->CTX_SUFF(pVM);
9360 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9361 rc = VBOXSTRICTRC_VAL(rc2);
9362 if (RT_LIKELY( rc == VINF_SUCCESS
9363 || rc == VINF_EM_HALT))
9364 {
9365 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9366 AssertRCReturn(rc3, rc3);
9367
9368 if ( rc == VINF_EM_HALT
9369 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9370 {
9371 rc = VINF_SUCCESS;
9372 }
9373 }
9374 else
9375 {
9376 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9377 rc = VERR_EM_INTERPRETER;
9378 }
9379 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9380 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9381 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9382 return rc;
9383}
9384
9385
9386/**
9387 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9388 */
9389HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9390{
9391 /*
9392 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9393 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9394 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9395 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9396 */
9397 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9398 HMVMX_RETURN_UNEXPECTED_EXIT();
9399}
9400
9401
9402/**
9403 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9404 */
9405HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9406{
9407 /*
9408 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9409 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9410 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9411 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9412 */
9413 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9414 HMVMX_RETURN_UNEXPECTED_EXIT();
9415}
9416
9417
9418/**
9419 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9420 */
9421HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9422{
9423 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9424 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9425 HMVMX_RETURN_UNEXPECTED_EXIT();
9426}
9427
9428
9429/**
9430 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9431 */
9432HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9433{
9434 /*
9435 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9436 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9437 * See Intel spec. 25.3 "Other Causes of VM-exits".
9438 */
9439 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9440 HMVMX_RETURN_UNEXPECTED_EXIT();
9441}
9442
9443
9444/**
9445 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9446 * VM-exit.
9447 */
9448HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9449{
9450 /*
9451 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9452 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9453 *
9454 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9455 * See Intel spec. "23.8 Restrictions on VMX operation".
9456 */
9457 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9458 return VINF_SUCCESS;
9459}
9460
9461
9462/**
9463 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9464 * VM-exit.
9465 */
9466HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9467{
9468 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9469 return VINF_EM_RESET;
9470}
9471
9472
9473/**
9474 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9475 */
9476HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9477{
9478 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9479 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9480 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9481 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9482 AssertRCReturn(rc, rc);
9483
9484 pMixedCtx->rip++;
9485 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9486 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9487 rc = VINF_SUCCESS;
9488 else
9489 rc = VINF_EM_HALT;
9490
9491 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9492 return rc;
9493}
9494
9495
9496/**
9497 * VM-exit handler for instructions that result in a #UD exception delivered to
9498 * the guest.
9499 */
9500HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9501{
9502 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9503 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9504 return VINF_SUCCESS;
9505}
9506
9507
9508/**
9509 * VM-exit handler for expiry of the VMX preemption timer.
9510 */
9511HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9512{
9513 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9514
9515 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9516 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9517
9518 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9519 PVM pVM = pVCpu->CTX_SUFF(pVM);
9520 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9521 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9522 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9523}
9524
9525
9526/**
9527 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9528 */
9529HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9530{
9531 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9532
9533 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9534 /** @todo check if XSETBV is supported by the recompiler. */
9535 return VERR_EM_INTERPRETER;
9536}
9537
9538
9539/**
9540 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9541 */
9542HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9543{
9544 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9545
9546 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9547 /** @todo implement EMInterpretInvpcid() */
9548 return VERR_EM_INTERPRETER;
9549}
9550
9551
9552/**
9553 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9554 * Error VM-exit.
9555 */
9556HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9557{
9558 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9559 AssertRCReturn(rc, rc);
9560
9561 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9562 NOREF(uInvalidReason);
9563
9564#ifdef VBOX_STRICT
9565 uint32_t uIntrState;
9566 HMVMXHCUINTREG uHCReg;
9567 uint64_t u64Val;
9568 uint32_t u32Val;
9569
9570 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9571 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9572 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9573 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9574 AssertRCReturn(rc, rc);
9575
9576 Log4(("uInvalidReason %u\n", uInvalidReason));
9577 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9578 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9579 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9580 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9581
9582 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9583 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9584 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9585 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9586 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9587 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9588 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9589 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9590 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9591 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9592 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9593 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9594#endif
9595
9596 PVM pVM = pVCpu->CTX_SUFF(pVM);
9597 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9598
9599 return VERR_VMX_INVALID_GUEST_STATE;
9600}
9601
9602
9603/**
9604 * VM-exit handler for VM-entry failure due to an MSR-load
9605 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9606 */
9607HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9608{
9609 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9610 HMVMX_RETURN_UNEXPECTED_EXIT();
9611}
9612
9613
9614/**
9615 * VM-exit handler for VM-entry failure due to a machine-check event
9616 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9617 */
9618HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9619{
9620 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9621 HMVMX_RETURN_UNEXPECTED_EXIT();
9622}
9623
9624
9625/**
9626 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9627 * theory.
9628 */
9629HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9630{
9631 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9632 return VERR_VMX_UNDEFINED_EXIT_CODE;
9633}
9634
9635
9636/**
9637 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9638 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9639 * Conditional VM-exit.
9640 */
9641HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9642{
9643 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9644
9645 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9646 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9647 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9648 return VERR_EM_INTERPRETER;
9649 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9650 HMVMX_RETURN_UNEXPECTED_EXIT();
9651}
9652
9653
9654/**
9655 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9656 */
9657HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9658{
9659 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9660
9661 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9662 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9663 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9664 return VERR_EM_INTERPRETER;
9665 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9666 HMVMX_RETURN_UNEXPECTED_EXIT();
9667}
9668
9669
9670/**
9671 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9672 */
9673HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9674{
9675 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9676
9677 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9678 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9679 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9680 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9681 AssertRCReturn(rc, rc);
9682 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9683
9684 PVM pVM = pVCpu->CTX_SUFF(pVM);
9685 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9686 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9687 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9689
9690 if (RT_LIKELY(rc == VINF_SUCCESS))
9691 {
9692 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9693 Assert(pVmxTransient->cbInstr == 2);
9694 }
9695 return rc;
9696}
9697
9698
9699/**
9700 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9701 */
9702HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9703{
9704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9705 PVM pVM = pVCpu->CTX_SUFF(pVM);
9706 int rc = VINF_SUCCESS;
9707
9708 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9709 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9710 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9711 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9712 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9713 AssertRCReturn(rc, rc);
9714 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9715
9716 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9717 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9718 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9719
9720 if (RT_LIKELY(rc == VINF_SUCCESS))
9721 {
9722 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9723
9724 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9725 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9726 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9727 {
9728 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9729 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9730 EMInterpretWrmsr() changes it. */
9731 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9732 }
9733 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9734 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9735 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9736 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9737
9738 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9739 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9740 {
9741 switch (pMixedCtx->ecx)
9742 {
9743 case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9744 case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9745 case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9746 case MSR_K8_FS_BASE: /* no break */
9747 case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9748 case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9749 }
9750 }
9751#ifdef VBOX_STRICT
9752 else
9753 {
9754 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9755 switch (pMixedCtx->ecx)
9756 {
9757 case MSR_IA32_SYSENTER_CS:
9758 case MSR_IA32_SYSENTER_EIP:
9759 case MSR_IA32_SYSENTER_ESP:
9760 case MSR_K8_FS_BASE:
9761 case MSR_K8_GS_BASE:
9762 {
9763 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9764 HMVMX_RETURN_UNEXPECTED_EXIT();
9765 }
9766
9767 case MSR_K8_LSTAR:
9768 case MSR_K6_STAR:
9769 case MSR_K8_SF_MASK:
9770 case MSR_K8_TSC_AUX:
9771 case MSR_K8_KERNEL_GS_BASE:
9772 {
9773 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9774 pMixedCtx->ecx));
9775 HMVMX_RETURN_UNEXPECTED_EXIT();
9776 }
9777 }
9778 }
9779#endif /* VBOX_STRICT */
9780 }
9781 return rc;
9782}
9783
9784
9785/**
9786 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9787 */
9788HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9789{
9790 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9791
9792 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9794 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9795 return VERR_EM_INTERPRETER;
9796 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9797 HMVMX_RETURN_UNEXPECTED_EXIT();
9798}
9799
9800
9801/**
9802 * VM-exit handler for when the TPR value is lowered below the specified
9803 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9804 */
9805HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9806{
9807 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9808 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9809
9810 /*
9811 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9812 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9813 * resume guest execution.
9814 */
9815 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9816 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9817 return VINF_SUCCESS;
9818}
9819
9820
9821/**
9822 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9823 * VM-exit.
9824 *
9825 * @retval VINF_SUCCESS when guest execution can continue.
9826 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9827 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9828 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9829 * recompiler.
9830 */
9831HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9832{
9833 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9834 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9835 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9836 AssertRCReturn(rc, rc);
9837
9838 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9839 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9840 PVM pVM = pVCpu->CTX_SUFF(pVM);
9841 switch (uAccessType)
9842 {
9843 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9844 {
9845#if 0
9846 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9847 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9848#else
9849 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9850 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9851 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9852#endif
9853 AssertRCReturn(rc, rc);
9854
9855 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9856 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9857 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9858 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9859
9860 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9861 {
9862 case 0: /* CR0 */
9863 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9864 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9865 break;
9866 case 2: /* CR2 */
9867 /* Nothing to do here, CR2 it's not part of the VMCS. */
9868 break;
9869 case 3: /* CR3 */
9870 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9871 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9872 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9873 break;
9874 case 4: /* CR4 */
9875 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9876 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9877 break;
9878 case 8: /* CR8 */
9879 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9880 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9881 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9882 break;
9883 default:
9884 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9885 break;
9886 }
9887
9888 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9889 break;
9890 }
9891
9892 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9893 {
9894 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9895 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9896 AssertRCReturn(rc, rc);
9897 Assert( !pVM->hm.s.fNestedPaging
9898 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9899 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9900
9901 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9902 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9903 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9904
9905 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9906 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9907 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9908 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9909 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9910 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9911 break;
9912 }
9913
9914 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9915 {
9916 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9917 AssertRCReturn(rc, rc);
9918 rc = EMInterpretCLTS(pVM, pVCpu);
9919 AssertRCReturn(rc, rc);
9920 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9922 Log4(("CRX CLTS write rc=%d\n", rc));
9923 break;
9924 }
9925
9926 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9927 {
9928 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9929 AssertRCReturn(rc, rc);
9930 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9931 if (RT_LIKELY(rc == VINF_SUCCESS))
9932 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9933 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9934 Log4(("CRX LMSW write rc=%d\n", rc));
9935 break;
9936 }
9937
9938 default:
9939 {
9940 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9941 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9942 }
9943 }
9944
9945 /* Validate possible error codes. */
9946 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9947 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9948 if (RT_SUCCESS(rc))
9949 {
9950 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9951 AssertRCReturn(rc2, rc2);
9952 }
9953
9954 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9955 return rc;
9956}
9957
9958
9959/**
9960 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9961 * VM-exit.
9962 */
9963HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9964{
9965 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9966 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9967
9968 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9969 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9970 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9971 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9972 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9973 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9974 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9975 AssertRCReturn(rc2, rc2);
9976
9977 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9978 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9979 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9980 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9981 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9982 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9983 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9984
9985 /* I/O operation lookup arrays. */
9986 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9987 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9988
9989 VBOXSTRICTRC rcStrict;
9990 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9991 const uint32_t cbInstr = pVmxTransient->cbInstr;
9992 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9993 PVM pVM = pVCpu->CTX_SUFF(pVM);
9994 if (fIOString)
9995 {
9996#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9997 /*
9998 * INS/OUTS - I/O String instruction.
9999 *
10000 * Use instruction-information if available, otherwise fall back on
10001 * interpreting the instruction.
10002 */
10003 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10004 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10005 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10006 {
10007 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
10008 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10009 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10010 AssertRCReturn(rc2, rc2);
10011 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10012 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10013 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10014 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10015 if (fIOWrite)
10016 {
10017 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10018 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10019 }
10020 else
10021 {
10022 /*
10023 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10024 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10025 * See Intel Instruction spec. for "INS".
10026 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10027 */
10028 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10029 }
10030 }
10031 else
10032 {
10033 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10034 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10035 AssertRCReturn(rc2, rc2);
10036 rcStrict = IEMExecOne(pVCpu);
10037 }
10038 /** @todo IEM needs to be setting these flags somehow. */
10039 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10040 fUpdateRipAlready = true;
10041#else
10042 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10043 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
10044 if (RT_SUCCESS(rcStrict))
10045 {
10046 if (fIOWrite)
10047 {
10048 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10049 (DISCPUMODE)pDis->uAddrMode, cbValue);
10050 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10051 }
10052 else
10053 {
10054 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10055 (DISCPUMODE)pDis->uAddrMode, cbValue);
10056 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10057 }
10058 }
10059 else
10060 {
10061 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10062 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10063 }
10064#endif
10065 }
10066 else
10067 {
10068 /*
10069 * IN/OUT - I/O instruction.
10070 */
10071 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10072 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10073 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10074 if (fIOWrite)
10075 {
10076 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10077 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10078 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10079 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10080 }
10081 else
10082 {
10083 uint32_t u32Result = 0;
10084 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10085 if (IOM_SUCCESS(rcStrict))
10086 {
10087 /* Save result of I/O IN instr. in AL/AX/EAX. */
10088 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10089 }
10090 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10091 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10093 }
10094 }
10095
10096 if (IOM_SUCCESS(rcStrict))
10097 {
10098 if (!fUpdateRipAlready)
10099 {
10100 pMixedCtx->rip += cbInstr;
10101 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10102 }
10103
10104 /* INS & OUTS with REP prefix modify RFLAGS. */
10105 if (fIOString)
10106 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10107
10108 /*
10109 * If any I/O breakpoints are armed, we need to check if one triggered
10110 * and take appropriate action.
10111 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10112 */
10113 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10114 AssertRCReturn(rc2, rc2);
10115
10116 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10117 * execution engines about whether hyper BPs and such are pending. */
10118 uint32_t const uDr7 = pMixedCtx->dr[7];
10119 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10120 && X86_DR7_ANY_RW_IO(uDr7)
10121 && (pMixedCtx->cr4 & X86_CR4_DE))
10122 || DBGFBpIsHwIoArmed(pVM)))
10123 {
10124 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10125
10126 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10127 VMMRZCallRing3Disable(pVCpu);
10128 HM_DISABLE_PREEMPT_IF_NEEDED();
10129
10130 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10131
10132 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10133 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10134 {
10135 /* Raise #DB. */
10136 if (fIsGuestDbgActive)
10137 ASMSetDR6(pMixedCtx->dr[6]);
10138 if (pMixedCtx->dr[7] != uDr7)
10139 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10140
10141 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10142 }
10143 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10144 else if ( rcStrict2 != VINF_SUCCESS
10145 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10146 rcStrict = rcStrict2;
10147
10148 HM_RESTORE_PREEMPT_IF_NEEDED();
10149 VMMRZCallRing3Enable(pVCpu);
10150 }
10151 }
10152
10153#ifdef DEBUG
10154 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10155 Assert(!fIOWrite);
10156 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10157 Assert(fIOWrite);
10158 else
10159 {
10160 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10161 * statuses, that the VMM device and some others may return. See
10162 * IOM_SUCCESS() for guidance. */
10163 AssertMsg( RT_FAILURE(rcStrict)
10164 || rcStrict == VINF_SUCCESS
10165 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10166 || rcStrict == VINF_EM_DBG_BREAKPOINT
10167 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10168 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10169 }
10170#endif
10171
10172 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10173 return VBOXSTRICTRC_TODO(rcStrict);
10174}
10175
10176
10177/**
10178 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10179 * VM-exit.
10180 */
10181HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10182{
10183 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10184
10185 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10186 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10187 AssertRCReturn(rc, rc);
10188 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10189 {
10190 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10191 AssertRCReturn(rc, rc);
10192 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10193 {
10194 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10195
10196 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10197 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10198 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10199 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10200 {
10201 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10202 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10203
10204 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10205 Assert(!pVCpu->hm.s.Event.fPending);
10206 pVCpu->hm.s.Event.fPending = true;
10207 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10208 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10209 AssertRCReturn(rc, rc);
10210 if (fErrorCodeValid)
10211 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10212 else
10213 pVCpu->hm.s.Event.u32ErrCode = 0;
10214 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10215 && uVector == X86_XCPT_PF)
10216 {
10217 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10218 }
10219
10220 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10221 }
10222 }
10223 }
10224
10225 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10226 * emulation. */
10227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10228 return VERR_EM_INTERPRETER;
10229}
10230
10231
10232/**
10233 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10234 */
10235HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10236{
10237 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10238 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10240 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10241 AssertRCReturn(rc, rc);
10242 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10243 return VINF_EM_DBG_STEPPED;
10244}
10245
10246
10247/**
10248 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10249 */
10250HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10251{
10252 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10253
10254 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10255 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10256 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10257 return VINF_SUCCESS;
10258 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10259 return rc;
10260
10261#if 0
10262 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10263 * just sync the whole thing. */
10264 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10265#else
10266 /* Aggressive state sync. for now. */
10267 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10268 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10269 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10270#endif
10271 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10272 AssertRCReturn(rc, rc);
10273
10274 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10275 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10276 switch (uAccessType)
10277 {
10278 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10279 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10280 {
10281 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10282 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10283 {
10284 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10285 }
10286
10287 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10288 GCPhys &= PAGE_BASE_GC_MASK;
10289 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10290 PVM pVM = pVCpu->CTX_SUFF(pVM);
10291 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10292 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10293
10294 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10295 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10296 CPUMCTX2CORE(pMixedCtx), GCPhys);
10297 rc = VBOXSTRICTRC_VAL(rc2);
10298 Log4(("ApicAccess rc=%d\n", rc));
10299 if ( rc == VINF_SUCCESS
10300 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10301 || rc == VERR_PAGE_NOT_PRESENT)
10302 {
10303 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10304 | HM_CHANGED_GUEST_RSP
10305 | HM_CHANGED_GUEST_RFLAGS
10306 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10307 rc = VINF_SUCCESS;
10308 }
10309 break;
10310 }
10311
10312 default:
10313 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10314 rc = VINF_EM_RAW_EMULATE_INSTR;
10315 break;
10316 }
10317
10318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10319 return rc;
10320}
10321
10322
10323/**
10324 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10325 * VM-exit.
10326 */
10327HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10328{
10329 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10330
10331 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10332 if (pVmxTransient->fWasGuestDebugStateActive)
10333 {
10334 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10335 HMVMX_RETURN_UNEXPECTED_EXIT();
10336 }
10337
10338 int rc = VERR_INTERNAL_ERROR_5;
10339 if ( !DBGFIsStepping(pVCpu)
10340 && !pVCpu->hm.s.fSingleInstruction
10341 && !pVmxTransient->fWasHyperDebugStateActive)
10342 {
10343 /* Don't intercept MOV DRx and #DB any more. */
10344 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10345 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10346 AssertRCReturn(rc, rc);
10347
10348 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10349 {
10350#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10351 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10352 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10353 AssertRCReturn(rc, rc);
10354#endif
10355 }
10356
10357 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10358 VMMRZCallRing3Disable(pVCpu);
10359 HM_DISABLE_PREEMPT_IF_NEEDED();
10360
10361 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10362 PVM pVM = pVCpu->CTX_SUFF(pVM);
10363 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10364 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10365
10366 HM_RESTORE_PREEMPT_IF_NEEDED();
10367 VMMRZCallRing3Enable(pVCpu);
10368
10369#ifdef VBOX_WITH_STATISTICS
10370 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10371 AssertRCReturn(rc, rc);
10372 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10373 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10374 else
10375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10376#endif
10377 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10378 return VINF_SUCCESS;
10379 }
10380
10381 /*
10382 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10383 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10384 */
10385 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10386 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10387 AssertRCReturn(rc, rc);
10388 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10389
10390 PVM pVM = pVCpu->CTX_SUFF(pVM);
10391 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10392 {
10393 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10394 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10395 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10396 if (RT_SUCCESS(rc))
10397 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10398 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10399 }
10400 else
10401 {
10402 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10403 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10404 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10405 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10406 }
10407
10408 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10409 if (RT_SUCCESS(rc))
10410 {
10411 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10412 AssertRCReturn(rc2, rc2);
10413 }
10414 return rc;
10415}
10416
10417
10418/**
10419 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10420 * Conditional VM-exit.
10421 */
10422HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10423{
10424 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10425 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10426
10427 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10428 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10429 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10430 return VINF_SUCCESS;
10431 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10432 return rc;
10433
10434 RTGCPHYS GCPhys = 0;
10435 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10436
10437#if 0
10438 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10439#else
10440 /* Aggressive state sync. for now. */
10441 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10442 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10443 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10444#endif
10445 AssertRCReturn(rc, rc);
10446
10447 /*
10448 * If we succeed, resume guest execution.
10449 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10450 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10451 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10452 * weird case. See @bugref{6043}.
10453 */
10454 PVM pVM = pVCpu->CTX_SUFF(pVM);
10455 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10456 rc = VBOXSTRICTRC_VAL(rc2);
10457 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10458 if ( rc == VINF_SUCCESS
10459 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10460 || rc == VERR_PAGE_NOT_PRESENT)
10461 {
10462 /* Successfully handled MMIO operation. */
10463 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10464 | HM_CHANGED_GUEST_RSP
10465 | HM_CHANGED_GUEST_RFLAGS
10466 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10467 rc = VINF_SUCCESS;
10468 }
10469 return rc;
10470}
10471
10472
10473/**
10474 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10475 * VM-exit.
10476 */
10477HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10478{
10479 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10480 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10481
10482 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10483 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10484 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10485 return VINF_SUCCESS;
10486 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10487 return rc;
10488
10489 RTGCPHYS GCPhys = 0;
10490 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10491 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10492#if 0
10493 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10494#else
10495 /* Aggressive state sync. for now. */
10496 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10497 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10498 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10499#endif
10500 AssertRCReturn(rc, rc);
10501
10502 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10503 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10504
10505 RTGCUINT uErrorCode = 0;
10506 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10507 uErrorCode |= X86_TRAP_PF_ID;
10508 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10509 uErrorCode |= X86_TRAP_PF_RW;
10510 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10511 uErrorCode |= X86_TRAP_PF_P;
10512
10513 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10514
10515 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10516 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10517
10518 /* Handle the pagefault trap for the nested shadow table. */
10519 PVM pVM = pVCpu->CTX_SUFF(pVM);
10520 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10521 TRPMResetTrap(pVCpu);
10522
10523 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10524 if ( rc == VINF_SUCCESS
10525 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10526 || rc == VERR_PAGE_NOT_PRESENT)
10527 {
10528 /* Successfully synced our nested page tables. */
10529 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10530 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10531 | HM_CHANGED_GUEST_RSP
10532 | HM_CHANGED_GUEST_RFLAGS);
10533 return VINF_SUCCESS;
10534 }
10535
10536 Log4(("EPT return to ring-3 rc=%d\n"));
10537 return rc;
10538}
10539
10540/** @} */
10541
10542/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10543/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10544/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10545
10546/** @name VM-exit exception handlers.
10547 * @{
10548 */
10549
10550/**
10551 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10552 */
10553static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10554{
10555 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10557
10558 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10559 AssertRCReturn(rc, rc);
10560
10561 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10562 {
10563 /* Old-style FPU error reporting needs some extra work. */
10564 /** @todo don't fall back to the recompiler, but do it manually. */
10565 return VERR_EM_INTERPRETER;
10566 }
10567
10568 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10569 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10570 return rc;
10571}
10572
10573
10574/**
10575 * VM-exit exception handler for #BP (Breakpoint exception).
10576 */
10577static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10578{
10579 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10580 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10581
10582 /** @todo Try optimize this by not saving the entire guest state unless
10583 * really needed. */
10584 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10585 AssertRCReturn(rc, rc);
10586
10587 PVM pVM = pVCpu->CTX_SUFF(pVM);
10588 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10589 if (rc == VINF_EM_RAW_GUEST_TRAP)
10590 {
10591 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10592 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10593 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10594 AssertRCReturn(rc, rc);
10595
10596 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10597 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10598 }
10599
10600 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10601 return rc;
10602}
10603
10604
10605/**
10606 * VM-exit exception handler for #DB (Debug exception).
10607 */
10608static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10609{
10610 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10611 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10612 Log6(("XcptDB\n"));
10613
10614 /*
10615 * Get the DR6-like values from the exit qualification and pass it to DBGF
10616 * for processing.
10617 */
10618 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10619 AssertRCReturn(rc, rc);
10620
10621 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10622 uint64_t uDR6 = X86_DR6_INIT_VAL;
10623 uDR6 |= ( pVmxTransient->uExitQualification
10624 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10625
10626 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10627 if (rc == VINF_EM_RAW_GUEST_TRAP)
10628 {
10629 /*
10630 * The exception was for the guest. Update DR6, DR7.GD and
10631 * IA32_DEBUGCTL.LBR before forwarding it.
10632 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10633 */
10634 VMMRZCallRing3Disable(pVCpu);
10635 HM_DISABLE_PREEMPT_IF_NEEDED();
10636
10637 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10638 pMixedCtx->dr[6] |= uDR6;
10639 if (CPUMIsGuestDebugStateActive(pVCpu))
10640 ASMSetDR6(pMixedCtx->dr[6]);
10641
10642 HM_RESTORE_PREEMPT_IF_NEEDED();
10643 VMMRZCallRing3Enable(pVCpu);
10644
10645 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10646 AssertRCReturn(rc, rc);
10647
10648 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10649 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10650
10651 /* Paranoia. */
10652 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10653 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10654
10655 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10656 AssertRCReturn(rc, rc);
10657
10658 /*
10659 * Raise #DB in the guest.
10660 */
10661 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10662 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10663 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10664 AssertRCReturn(rc, rc);
10665 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10666 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10667 return VINF_SUCCESS;
10668 }
10669
10670 /*
10671 * Not a guest trap, must be a hypervisor related debug event then.
10672 * Update DR6 in case someone is interested in it.
10673 */
10674 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10675 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10676 CPUMSetHyperDR6(pVCpu, uDR6);
10677
10678 return rc;
10679}
10680
10681
10682/**
10683 * VM-exit exception handler for #NM (Device-not-available exception: floating
10684 * point exception).
10685 */
10686static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10687{
10688 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10689
10690 /* We require CR0 and EFER. EFER is always up-to-date. */
10691 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10692 AssertRCReturn(rc, rc);
10693
10694 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10695 VMMRZCallRing3Disable(pVCpu);
10696 HM_DISABLE_PREEMPT_IF_NEEDED();
10697
10698 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10699 if (pVmxTransient->fWasGuestFPUStateActive)
10700 {
10701 rc = VINF_EM_RAW_GUEST_TRAP;
10702 Assert(CPUMIsGuestFPUStateActive(pVCpu) || VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10703 }
10704 else
10705 {
10706#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10707 Assert(!pVmxTransient->fWasGuestFPUStateActive);
10708#endif
10709 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10710 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10711 }
10712
10713 HM_RESTORE_PREEMPT_IF_NEEDED();
10714 VMMRZCallRing3Enable(pVCpu);
10715
10716 if (rc == VINF_SUCCESS)
10717 {
10718 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10719 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10720 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10721 pVCpu->hm.s.fUseGuestFpu = true;
10722 }
10723 else
10724 {
10725 /* Forward #NM to the guest. */
10726 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10727 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10728 AssertRCReturn(rc, rc);
10729 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10730 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10731 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10732 }
10733
10734 return VINF_SUCCESS;
10735}
10736
10737
10738/**
10739 * VM-exit exception handler for #GP (General-protection exception).
10740 *
10741 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10742 */
10743static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10744{
10745 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10746 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10747
10748 int rc = VERR_INTERNAL_ERROR_5;
10749 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10750 {
10751#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10752 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10753 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10754 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10755 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10756 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10757 AssertRCReturn(rc, rc);
10758 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10759 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10760 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10761 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10762 return rc;
10763#else
10764 /* We don't intercept #GP. */
10765 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10766 return VERR_VMX_UNEXPECTED_EXCEPTION;
10767#endif
10768 }
10769
10770 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10771 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10772
10773 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10774 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10775 AssertRCReturn(rc, rc);
10776
10777 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10778 uint32_t cbOp = 0;
10779 PVM pVM = pVCpu->CTX_SUFF(pVM);
10780 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10781 if (RT_SUCCESS(rc))
10782 {
10783 rc = VINF_SUCCESS;
10784 Assert(cbOp == pDis->cbInstr);
10785 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10786 switch (pDis->pCurInstr->uOpcode)
10787 {
10788 case OP_CLI:
10789 {
10790 pMixedCtx->eflags.Bits.u1IF = 0;
10791 pMixedCtx->rip += pDis->cbInstr;
10792 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10794 break;
10795 }
10796
10797 case OP_STI:
10798 {
10799 pMixedCtx->eflags.Bits.u1IF = 1;
10800 pMixedCtx->rip += pDis->cbInstr;
10801 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10802 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10803 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10804 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10805 break;
10806 }
10807
10808 case OP_HLT:
10809 {
10810 rc = VINF_EM_HALT;
10811 pMixedCtx->rip += pDis->cbInstr;
10812 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10814 break;
10815 }
10816
10817 case OP_POPF:
10818 {
10819 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10820 uint32_t cbParm = 0;
10821 uint32_t uMask = 0;
10822 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10823 {
10824 cbParm = 4;
10825 uMask = 0xffffffff;
10826 }
10827 else
10828 {
10829 cbParm = 2;
10830 uMask = 0xffff;
10831 }
10832
10833 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10834 RTGCPTR GCPtrStack = 0;
10835 X86EFLAGS Eflags;
10836 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10837 &GCPtrStack);
10838 if (RT_SUCCESS(rc))
10839 {
10840 Assert(sizeof(Eflags.u32) >= cbParm);
10841 Eflags.u32 = 0;
10842 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10843 }
10844 if (RT_FAILURE(rc))
10845 {
10846 rc = VERR_EM_INTERPRETER;
10847 break;
10848 }
10849 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10850 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10851 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10852 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10853 pMixedCtx->esp += cbParm;
10854 pMixedCtx->esp &= uMask;
10855 pMixedCtx->rip += pDis->cbInstr;
10856
10857 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10858 | HM_CHANGED_GUEST_RSP
10859 | HM_CHANGED_GUEST_RFLAGS);
10860 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10861 break;
10862 }
10863
10864 case OP_PUSHF:
10865 {
10866 uint32_t cbParm = 0;
10867 uint32_t uMask = 0;
10868 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10869 {
10870 cbParm = 4;
10871 uMask = 0xffffffff;
10872 }
10873 else
10874 {
10875 cbParm = 2;
10876 uMask = 0xffff;
10877 }
10878
10879 /* Get the stack pointer & push the contents of eflags onto the stack. */
10880 RTGCPTR GCPtrStack = 0;
10881 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10882 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10883 if (RT_FAILURE(rc))
10884 {
10885 rc = VERR_EM_INTERPRETER;
10886 break;
10887 }
10888 X86EFLAGS Eflags = pMixedCtx->eflags;
10889 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10890 Eflags.Bits.u1RF = 0;
10891 Eflags.Bits.u1VM = 0;
10892
10893 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10894 if (RT_FAILURE(rc))
10895 {
10896 rc = VERR_EM_INTERPRETER;
10897 break;
10898 }
10899 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10900 pMixedCtx->esp -= cbParm;
10901 pMixedCtx->esp &= uMask;
10902 pMixedCtx->rip += pDis->cbInstr;
10903 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
10904 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10905 break;
10906 }
10907
10908 case OP_IRET:
10909 {
10910 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10911 * instruction reference. */
10912 RTGCPTR GCPtrStack = 0;
10913 uint32_t uMask = 0xffff;
10914 uint16_t aIretFrame[3];
10915 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10916 {
10917 rc = VERR_EM_INTERPRETER;
10918 break;
10919 }
10920 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10921 &GCPtrStack);
10922 if (RT_SUCCESS(rc))
10923 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10924 if (RT_FAILURE(rc))
10925 {
10926 rc = VERR_EM_INTERPRETER;
10927 break;
10928 }
10929 pMixedCtx->eip = 0;
10930 pMixedCtx->ip = aIretFrame[0];
10931 pMixedCtx->cs.Sel = aIretFrame[1];
10932 pMixedCtx->cs.ValidSel = aIretFrame[1];
10933 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10934 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10935 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10936 pMixedCtx->sp += sizeof(aIretFrame);
10937 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10938 | HM_CHANGED_GUEST_SEGMENT_REGS
10939 | HM_CHANGED_GUEST_RSP
10940 | HM_CHANGED_GUEST_RFLAGS);
10941 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10942 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10943 break;
10944 }
10945
10946 case OP_INT:
10947 {
10948 uint16_t uVector = pDis->Param1.uValue & 0xff;
10949 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10951 break;
10952 }
10953
10954 case OP_INTO:
10955 {
10956 if (pMixedCtx->eflags.Bits.u1OF)
10957 {
10958 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10960 }
10961 break;
10962 }
10963
10964 default:
10965 {
10966 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10967 EMCODETYPE_SUPERVISOR);
10968 rc = VBOXSTRICTRC_VAL(rc2);
10969 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10970 Log4(("#GP rc=%Rrc\n", rc));
10971 break;
10972 }
10973 }
10974 }
10975 else
10976 rc = VERR_EM_INTERPRETER;
10977
10978 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10979 ("#GP Unexpected rc=%Rrc\n", rc));
10980 return rc;
10981}
10982
10983
10984/**
10985 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10986 * the exception reported in the VMX transient structure back into the VM.
10987 *
10988 * @remarks Requires uExitIntInfo in the VMX transient structure to be
10989 * up-to-date.
10990 */
10991static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10992{
10993 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10994
10995 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10996 hmR0VmxCheckExitDueToEventDelivery(). */
10997 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10998 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10999 AssertRCReturn(rc, rc);
11000 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11001
11002 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11003 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11004 return VINF_SUCCESS;
11005}
11006
11007
11008/**
11009 * VM-exit exception handler for #PF (Page-fault exception).
11010 */
11011static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11012{
11013 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11014 PVM pVM = pVCpu->CTX_SUFF(pVM);
11015 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11016 rc |= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
11017 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11018 AssertRCReturn(rc, rc);
11019
11020#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11021 if (pVM->hm.s.fNestedPaging)
11022 {
11023 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11024 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11025 {
11026 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11027 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11028 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11029 }
11030 else
11031 {
11032 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11033 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11034 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11035 }
11036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11037 return rc;
11038 }
11039#else
11040 Assert(!pVM->hm.s.fNestedPaging);
11041#endif
11042
11043 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11044 AssertRCReturn(rc, rc);
11045
11046 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11047 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11048
11049 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11050 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11051 (RTGCPTR)pVmxTransient->uExitQualification);
11052
11053 Log4(("#PF: rc=%Rrc\n", rc));
11054 if (rc == VINF_SUCCESS)
11055 {
11056 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11057 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11058 * memory? We don't update the whole state here... */
11059 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11060 | HM_CHANGED_GUEST_RSP
11061 | HM_CHANGED_GUEST_RFLAGS
11062 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11063 TRPMResetTrap(pVCpu);
11064 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11065 return rc;
11066 }
11067 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11068 {
11069 if (!pVmxTransient->fVectoringPF)
11070 {
11071 /* It's a guest page fault and needs to be reflected to the guest. */
11072 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11073 TRPMResetTrap(pVCpu);
11074 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11075 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11076 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11077 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11078 }
11079 else
11080 {
11081 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11082 TRPMResetTrap(pVCpu);
11083 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11084 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11085 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11086 }
11087
11088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11089 return VINF_SUCCESS;
11090 }
11091
11092 TRPMResetTrap(pVCpu);
11093 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11094 return rc;
11095}
11096
11097/** @} */
11098
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette