VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49283

Last change on this file since 49283 was 49283, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Macro naming fix and use RT_BOOL.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 461.3 KB
Line 
1/* $Id: HMVMXR0.cpp 49283 2013-10-25 08:49:06Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#define HMVMX_ALWAYS_FLUSH_TLB
46#endif
47
48
49/*******************************************************************************
50* Defined Constants And Macros *
51*******************************************************************************/
52#if defined(RT_ARCH_AMD64)
53# define HMVMX_IS_64BIT_HOST_MODE() (true)
54typedef RTHCUINTREG HMVMXHCUINTREG;
55#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56extern "C" uint32_t g_fVMXIs64bitHost;
57# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58typedef uint64_t HMVMXHCUINTREG;
59#else
60# define HMVMX_IS_64BIT_HOST_MODE() (false)
61typedef RTHCUINTREG HMVMXHCUINTREG;
62#endif
63
64/** Use the function table. */
65#define HMVMX_USE_FUNCTION_TABLE
66
67/**
68 * The maximum number of MSRs we are willing to swap during a world-switch.
69 * Intel claims 512/check capability MSR, we don't want to do anywhere close
70 * to that. See Intel spec. 24.7.2 "VM-Exit Controls for MSRs"
71 *
72 * Bump this count as and when required, there's no backward compatibility
73 * requirement.
74 */
75#define HMVMX_MAX_SWAP_MSR_COUNT 5
76
77/** Determine which tagged-TLB flush handler to use. */
78#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
79#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
80#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
81#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
82
83/** @name Updated-guest-state flags.
84 * @{ */
85#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
86#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
87#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
88#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
89#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
90#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
91#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
92#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
93#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
94#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
95#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
96#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
97#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
98#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
99#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
100#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
101#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
102#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
103#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
104#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
105#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
106 | HMVMX_UPDATED_GUEST_RSP \
107 | HMVMX_UPDATED_GUEST_RFLAGS \
108 | HMVMX_UPDATED_GUEST_CR0 \
109 | HMVMX_UPDATED_GUEST_CR3 \
110 | HMVMX_UPDATED_GUEST_CR4 \
111 | HMVMX_UPDATED_GUEST_GDTR \
112 | HMVMX_UPDATED_GUEST_IDTR \
113 | HMVMX_UPDATED_GUEST_LDTR \
114 | HMVMX_UPDATED_GUEST_TR \
115 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
116 | HMVMX_UPDATED_GUEST_DEBUG \
117 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
118 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
119 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
120 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
121 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
122 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
123 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
124 | HMVMX_UPDATED_GUEST_APIC_STATE)
125/** @} */
126
127/** @name
128 * Flags to skip redundant reads of some common VMCS fields that are not part of
129 * the guest-CPU state but are in the transient structure.
130 */
131#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
132#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
133#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
134#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
135#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
136#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
137#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
138/** @} */
139
140/** @name
141 * States of the VMCS.
142 *
143 * This does not reflect all possible VMCS states but currently only those
144 * needed for maintaining the VMCS consistently even when thread-context hooks
145 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
146 */
147#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
148#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
149#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
150/** @} */
151
152/**
153 * Exception bitmap mask for real-mode guests (real-on-v86).
154 *
155 * We need to intercept all exceptions manually (except #PF). #NM is also
156 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
157 * even in real-mode if we have Nested Paging support.
158 */
159#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
160 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
161 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
162 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
163 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
164 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
165 | RT_BIT(X86_XCPT_XF))
166
167/**
168 * Exception bitmap mask for all contributory exceptions.
169 *
170 * Page fault is deliberately excluded here as it's conditional as to whether
171 * it's contributory or benign. Page faults are handled separately.
172 */
173#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
174 | RT_BIT(X86_XCPT_DE))
175
176/** Maximum VM-instruction error number. */
177#define HMVMX_INSTR_ERROR_MAX 28
178
179/** Profiling macro. */
180#ifdef HM_PROFILE_EXIT_DISPATCH
181# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
183#else
184# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
185# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
186#endif
187
188/** Assert that preemption is disabled or covered by thread-context hooks. */
189#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
191
192/** Assert that we haven't migrated CPUs when thread-context hooks are not
193 * used. */
194#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
195 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
196 ("Illegal migration! Entered on CPU %u Current %u\n", \
197 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
198
199/** Helper macro for VM-exit handlers called unexpectedly. */
200#define HMVMX_RETURN_UNEXPECTED_EXIT() \
201 do { \
202 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
203 return VERR_VMX_UNEXPECTED_EXIT; \
204 } while (0)
205
206
207/*******************************************************************************
208* Structures and Typedefs *
209*******************************************************************************/
210/**
211 * VMX transient state.
212 *
213 * A state structure for holding miscellaneous information across
214 * VMX non-root operation and restored after the transition.
215 */
216typedef struct VMXTRANSIENT
217{
218 /** The host's rflags/eflags. */
219 RTCCUINTREG uEflags;
220#if HC_ARCH_BITS == 32
221 uint32_t u32Alignment0;
222#endif
223 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
224 uint64_t u64LStarMsr;
225 /** The guest's TPR value used for TPR shadowing. */
226 uint8_t u8GuestTpr;
227 /** Alignment. */
228 uint8_t abAlignment0[7];
229
230 /** The basic VM-exit reason. */
231 uint16_t uExitReason;
232 /** Alignment. */
233 uint16_t u16Alignment0;
234 /** The VM-exit interruption error code. */
235 uint32_t uExitIntErrorCode;
236 /** The VM-exit exit qualification. */
237 uint64_t uExitQualification;
238
239 /** The VM-exit interruption-information field. */
240 uint32_t uExitIntInfo;
241 /** The VM-exit instruction-length field. */
242 uint32_t cbInstr;
243 /** The VM-exit instruction-information field. */
244 union
245 {
246 /** Plain unsigned int representation. */
247 uint32_t u;
248 /** INS and OUTS information. */
249 struct
250 {
251 uint32_t u6Reserved0 : 7;
252 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253 uint32_t u3AddrSize : 3;
254 uint32_t u5Reserved1 : 5;
255 /** The segment register (X86_SREG_XXX). */
256 uint32_t iSegReg : 3;
257 uint32_t uReserved2 : 14;
258 } StrIo;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest FPU was active at the time of VM-exit. */
281 bool fWasGuestFPUStateActive;
282 /** Whether the guest debug state was active at the time of VM-exit. */
283 bool fWasGuestDebugStateActive;
284 /** Whether the hyper debug state was active at the time of VM-exit. */
285 bool fWasHyperDebugStateActive;
286 /** Whether TSC-offsetting should be setup before VM-entry. */
287 bool fUpdateTscOffsettingAndPreemptTimer;
288 /** Whether the VM-exit was caused by a page-fault during delivery of a
289 * contributory exception or a page-fault. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322
323/**
324 * VMX VM-exit handler.
325 *
326 * @returns VBox status code.
327 * @param pVCpu Pointer to the VMCPU.
328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
329 * out-of-sync. Make sure to update the required
330 * fields before using them.
331 * @param pVmxTransient Pointer to the VMX-transient structure.
332 */
333#ifndef HMVMX_USE_FUNCTION_TABLE
334typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335#else
336typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
337/** Pointer to VM-exit handler. */
338typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
339#endif
340
341
342/*******************************************************************************
343* Internal Functions *
344*******************************************************************************/
345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
346static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
347static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
348static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
350#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
351static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
352#endif
353#ifndef HMVMX_USE_FUNCTION_TABLE
354DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
355# define HMVMX_EXIT_DECL static int
356#else
357# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
358#endif
359
360/** @name VM-exit handlers.
361 * @{
362 */
363static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
364static FNVMXEXITHANDLER hmR0VmxExitExtInt;
365static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
366static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
367static FNVMXEXITHANDLER hmR0VmxExitSipi;
368static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
369static FNVMXEXITHANDLER hmR0VmxExitSmi;
370static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
371static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
372static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
373static FNVMXEXITHANDLER hmR0VmxExitCpuid;
374static FNVMXEXITHANDLER hmR0VmxExitGetsec;
375static FNVMXEXITHANDLER hmR0VmxExitHlt;
376static FNVMXEXITHANDLER hmR0VmxExitInvd;
377static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
378static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
379static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
380static FNVMXEXITHANDLER hmR0VmxExitRsm;
381static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
382static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
383static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
384static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
385static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
386static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
387static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
388static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
389static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
390static FNVMXEXITHANDLER hmR0VmxExitMwait;
391static FNVMXEXITHANDLER hmR0VmxExitMtf;
392static FNVMXEXITHANDLER hmR0VmxExitMonitor;
393static FNVMXEXITHANDLER hmR0VmxExitPause;
394static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
395static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
396static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
397static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
398static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
404static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407/** @} */
408
409static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
417
418/*******************************************************************************
419* Global Variables *
420*******************************************************************************/
421#ifdef HMVMX_USE_FUNCTION_TABLE
422
423/**
424 * VMX_EXIT dispatch table.
425 */
426static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
427{
428 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
429 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
430 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
431 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
432 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
433 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
434 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
435 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
436 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
437 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
438 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
439 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
440 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
441 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
442 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
443 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
444 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
445 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
446 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
447 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
448 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
449 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
450 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
451 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
452 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
453 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
454 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
455 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
456 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
457 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
458 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
459 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
460 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
461 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
462 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
463 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
464 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
465 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
466 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
468 /* 40 UNDEFINED */ hmR0VmxExitPause,
469 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
470 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
471 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
472 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
473 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
474 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
475 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
476 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
477 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
478 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
479 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
480 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
481 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
482 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
483 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
484 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
485 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
486 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
487 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
488};
489#endif /* HMVMX_USE_FUNCTION_TABLE */
490
491#ifdef VBOX_STRICT
492static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
493{
494 /* 0 */ "(Not Used)",
495 /* 1 */ "VMCALL executed in VMX root operation.",
496 /* 2 */ "VMCLEAR with invalid physical address.",
497 /* 3 */ "VMCLEAR with VMXON pointer.",
498 /* 4 */ "VMLAUNCH with non-clear VMCS.",
499 /* 5 */ "VMRESUME with non-launched VMCS.",
500 /* 6 */ "VMRESUME after VMXOFF",
501 /* 7 */ "VM entry with invalid control fields.",
502 /* 8 */ "VM entry with invalid host state fields.",
503 /* 9 */ "VMPTRLD with invalid physical address.",
504 /* 10 */ "VMPTRLD with VMXON pointer.",
505 /* 11 */ "VMPTRLD with incorrect revision identifier.",
506 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
507 /* 13 */ "VMWRITE to read-only VMCS component.",
508 /* 14 */ "(Not Used)",
509 /* 15 */ "VMXON executed in VMX root operation.",
510 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
511 /* 17 */ "VM entry with non-launched executing VMCS.",
512 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
513 /* 19 */ "VMCALL with non-clear VMCS.",
514 /* 20 */ "VMCALL with invalid VM-exit control fields.",
515 /* 21 */ "(Not Used)",
516 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
517 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
518 /* 24 */ "VMCALL with invalid SMM-monitor features.",
519 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
520 /* 26 */ "VM entry with events blocked by MOV SS.",
521 /* 27 */ "(Not Used)",
522 /* 28 */ "Invalid operand to INVEPT/INVVPID."
523};
524#endif /* VBOX_STRICT */
525
526
527
528/**
529 * Updates the VM's last error record. If there was a VMX instruction error,
530 * reads the error data from the VMCS and updates VCPU's last error record as
531 * well.
532 *
533 * @param pVM Pointer to the VM.
534 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
535 * VERR_VMX_UNABLE_TO_START_VM or
536 * VERR_VMX_INVALID_VMCS_FIELD).
537 * @param rc The error code.
538 */
539static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
540{
541 AssertPtr(pVM);
542 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
543 || rc == VERR_VMX_UNABLE_TO_START_VM)
544 {
545 AssertPtrReturnVoid(pVCpu);
546 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
547 }
548 pVM->hm.s.lLastError = rc;
549}
550
551
552/**
553 * Reads the VM-entry interruption-information field from the VMCS into the VMX
554 * transient structure.
555 *
556 * @returns VBox status code.
557 * @param pVmxTransient Pointer to the VMX transient structure.
558 *
559 * @remarks No-long-jump zone!!!
560 */
561DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
562{
563 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
564 AssertRCReturn(rc, rc);
565 return VINF_SUCCESS;
566}
567
568
569/**
570 * Reads the VM-entry exception error code field from the VMCS into
571 * the VMX transient structure.
572 *
573 * @returns VBox status code.
574 * @param pVmxTransient Pointer to the VMX transient structure.
575 *
576 * @remarks No-long-jump zone!!!
577 */
578DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
579{
580 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
581 AssertRCReturn(rc, rc);
582 return VINF_SUCCESS;
583}
584
585
586/**
587 * Reads the VM-entry exception error code field from the VMCS into
588 * the VMX transient structure.
589 *
590 * @returns VBox status code.
591 * @param pVCpu Pointer to the VMCPU.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-exit interruption-information field from the VMCS into the VMX
606 * transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVCpu Pointer to the VMCPU.
610 * @param pVmxTransient Pointer to the VMX transient structure.
611 */
612DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
613{
614 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
615 {
616 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
617 AssertRCReturn(rc, rc);
618 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
619 }
620 return VINF_SUCCESS;
621}
622
623
624/**
625 * Reads the VM-exit interruption error code from the VMCS into the VMX
626 * transient structure.
627 *
628 * @returns VBox status code.
629 * @param pVCpu Pointer to the VMCPU.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 */
632DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
633{
634 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
635 {
636 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
637 AssertRCReturn(rc, rc);
638 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
639 }
640 return VINF_SUCCESS;
641}
642
643
644/**
645 * Reads the VM-exit instruction length field from the VMCS into the VMX
646 * transient structure.
647 *
648 * @returns VBox status code.
649 * @param pVCpu Pointer to the VMCPU.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
657 AssertRCReturn(rc, rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit instruction-information field from the VMCS into
666 * the VMX transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVCpu The cross context per CPU structure.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit qualification from the VMCS into the VMX transient structure.
686 *
687 * @returns VBox status code.
688 * @param pVCpu Pointer to the VMCPU.
689 * @param pVmxTransient Pointer to the VMX transient structure.
690 */
691DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
692{
693 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
694 {
695 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
696 AssertRCReturn(rc, rc);
697 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
698 }
699 return VINF_SUCCESS;
700}
701
702
703/**
704 * Reads the IDT-vectoring information field from the VMCS into the VMX
705 * transient structure.
706 *
707 * @returns VBox status code.
708 * @param pVmxTransient Pointer to the VMX transient structure.
709 *
710 * @remarks No-long-jump zone!!!
711 */
712DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
713{
714 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
715 {
716 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
717 AssertRCReturn(rc, rc);
718 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
719 }
720 return VINF_SUCCESS;
721}
722
723
724/**
725 * Reads the IDT-vectoring error code from the VMCS into the VMX
726 * transient structure.
727 *
728 * @returns VBox status code.
729 * @param pVmxTransient Pointer to the VMX transient structure.
730 */
731DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
732{
733 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
734 {
735 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
736 AssertRCReturn(rc, rc);
737 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
738 }
739 return VINF_SUCCESS;
740}
741
742
743/**
744 * Enters VMX root mode operation on the current CPU.
745 *
746 * @returns VBox status code.
747 * @param pVM Pointer to the VM (optional, can be NULL, after
748 * a resume).
749 * @param HCPhysCpuPage Physical address of the VMXON region.
750 * @param pvCpuPage Pointer to the VMXON region.
751 */
752static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
753{
754 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
755 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 if (pVM)
759 {
760 /* Write the VMCS revision dword to the VMXON region. */
761 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
762 }
763
764 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
765 RTCCUINTREG uEflags = ASMIntDisableFlags();
766
767 /* Enable the VMX bit in CR4 if necessary. */
768 RTCCUINTREG uCr4 = ASMGetCR4();
769 if (!(uCr4 & X86_CR4_VMXE))
770 ASMSetCR4(uCr4 | X86_CR4_VMXE);
771
772 /* Enter VMX root mode. */
773 int rc = VMXEnable(HCPhysCpuPage);
774 if (RT_FAILURE(rc))
775 ASMSetCR4(uCr4);
776
777 /* Restore interrupts. */
778 ASMSetFlags(uEflags);
779 return rc;
780}
781
782
783/**
784 * Exits VMX root mode operation on the current CPU.
785 *
786 * @returns VBox status code.
787 */
788static int hmR0VmxLeaveRootMode(void)
789{
790 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
791
792 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
793 RTCCUINTREG uEflags = ASMIntDisableFlags();
794
795 /* If we're for some reason not in VMX root mode, then don't leave it. */
796 RTCCUINTREG uHostCR4 = ASMGetCR4();
797
798 int rc;
799 if (uHostCR4 & X86_CR4_VMXE)
800 {
801 /* Exit VMX root mode and clear the VMX bit in CR4. */
802 VMXDisable();
803 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
804 rc = VINF_SUCCESS;
805 }
806 else
807 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
808
809 /* Restore interrupts. */
810 ASMSetFlags(uEflags);
811 return rc;
812}
813
814
815/**
816 * Allocates and maps one physically contiguous page. The allocated page is
817 * zero'd out. (Used by various VT-x structures).
818 *
819 * @returns IPRT status code.
820 * @param pMemObj Pointer to the ring-0 memory object.
821 * @param ppVirt Where to store the virtual address of the
822 * allocation.
823 * @param pPhys Where to store the physical address of the
824 * allocation.
825 */
826DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
827{
828 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
829 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
830 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
831
832 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
833 if (RT_FAILURE(rc))
834 return rc;
835 *ppVirt = RTR0MemObjAddress(*pMemObj);
836 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
837 ASMMemZero32(*ppVirt, PAGE_SIZE);
838 return VINF_SUCCESS;
839}
840
841
842/**
843 * Frees and unmaps an allocated physical page.
844 *
845 * @param pMemObj Pointer to the ring-0 memory object.
846 * @param ppVirt Where to re-initialize the virtual address of
847 * allocation as 0.
848 * @param pHCPhys Where to re-initialize the physical address of the
849 * allocation as 0.
850 */
851DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
852{
853 AssertPtr(pMemObj);
854 AssertPtr(ppVirt);
855 AssertPtr(pHCPhys);
856 if (*pMemObj != NIL_RTR0MEMOBJ)
857 {
858 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
859 AssertRC(rc);
860 *pMemObj = NIL_RTR0MEMOBJ;
861 *ppVirt = 0;
862 *pHCPhys = 0;
863 }
864}
865
866
867/**
868 * Worker function to free VT-x related structures.
869 *
870 * @returns IPRT status code.
871 * @param pVM Pointer to the VM.
872 */
873static void hmR0VmxStructsFree(PVM pVM)
874{
875 for (VMCPUID i = 0; i < pVM->cCpus; i++)
876 {
877 PVMCPU pVCpu = &pVM->aCpus[i];
878 AssertPtr(pVCpu);
879
880#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
881 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
883#endif
884
885 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
887
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
890 }
891
892 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
893#ifdef VBOX_WITH_CRASHDUMP_MAGIC
894 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
895#endif
896}
897
898
899/**
900 * Worker function to allocate VT-x related VM structures.
901 *
902 * @returns IPRT status code.
903 * @param pVM Pointer to the VM.
904 */
905static int hmR0VmxStructsAlloc(PVM pVM)
906{
907 /*
908 * Initialize members up-front so we can cleanup properly on allocation failure.
909 */
910#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
911 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
912 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
913 pVM->hm.s.vmx.HCPhys##a_Name = 0;
914
915#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
919
920#ifdef VBOX_WITH_CRASHDUMP_MAGIC
921 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
922#endif
923 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
924
925 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
926 for (VMCPUID i = 0; i < pVM->cCpus; i++)
927 {
928 PVMCPU pVCpu = &pVM->aCpus[i];
929 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
931 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
932#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935#endif
936 }
937#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
938#undef VMXLOCAL_INIT_VM_MEMOBJ
939
940 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
941 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
942 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
943 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
944
945 /*
946 * Allocate all the VT-x structures.
947 */
948 int rc = VINF_SUCCESS;
949#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951 if (RT_FAILURE(rc))
952 goto cleanup;
953 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
954 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
955#endif
956
957 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
958 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
959 {
960 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
961 &pVM->hm.s.vmx.HCPhysApicAccess);
962 if (RT_FAILURE(rc))
963 goto cleanup;
964 }
965
966 /*
967 * Initialize per-VCPU VT-x structures.
968 */
969 for (VMCPUID i = 0; i < pVM->cCpus; i++)
970 {
971 PVMCPU pVCpu = &pVM->aCpus[i];
972 AssertPtr(pVCpu);
973
974 /* Allocate the VM control structure (VMCS). */
975 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978
979 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
980 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
981 {
982 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
983 &pVCpu->hm.s.vmx.HCPhysVirtApic);
984 if (RT_FAILURE(rc))
985 goto cleanup;
986 }
987
988 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
990 {
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
992 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
996 }
997
998#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
999 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003
1004 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1005 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008#endif
1009 }
1010
1011 return VINF_SUCCESS;
1012
1013cleanup:
1014 hmR0VmxStructsFree(pVM);
1015 return rc;
1016}
1017
1018
1019/**
1020 * Does global VT-x initialization (called during module initialization).
1021 *
1022 * @returns VBox status code.
1023 */
1024VMMR0DECL(int) VMXR0GlobalInit(void)
1025{
1026#ifdef HMVMX_USE_FUNCTION_TABLE
1027 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1028# ifdef VBOX_STRICT
1029 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1030 Assert(g_apfnVMExitHandlers[i]);
1031# endif
1032#endif
1033 return VINF_SUCCESS;
1034}
1035
1036
1037/**
1038 * Does global VT-x termination (called during module termination).
1039 */
1040VMMR0DECL(void) VMXR0GlobalTerm()
1041{
1042 /* Nothing to do currently. */
1043}
1044
1045
1046/**
1047 * Sets up and activates VT-x on the current CPU.
1048 *
1049 * @returns VBox status code.
1050 * @param pCpu Pointer to the global CPU info struct.
1051 * @param pVM Pointer to the VM (can be NULL after a host resume
1052 * operation).
1053 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1054 * fEnabledByHost is true).
1055 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1056 * @a fEnabledByHost is true).
1057 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1058 * enable VT-x on the host.
1059 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1060 */
1061VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1062 void *pvMsrs)
1063{
1064 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1065 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067
1068 /* Enable VT-x if it's not already enabled by the host. */
1069 if (!fEnabledByHost)
1070 {
1071 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1072 if (RT_FAILURE(rc))
1073 return rc;
1074 }
1075
1076 /*
1077 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1078 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1079 */
1080 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1081 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1082 {
1083 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1084 pCpu->fFlushAsidBeforeUse = false;
1085 }
1086 else
1087 pCpu->fFlushAsidBeforeUse = true;
1088
1089 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1090 ++pCpu->cTlbFlushes;
1091
1092 return VINF_SUCCESS;
1093}
1094
1095
1096/**
1097 * Deactivates VT-x on the current CPU.
1098 *
1099 * @returns VBox status code.
1100 * @param pCpu Pointer to the global CPU info struct.
1101 * @param pvCpuPage Pointer to the VMXON region.
1102 * @param HCPhysCpuPage Physical address of the VMXON region.
1103 *
1104 * @remarks This function should never be called when SUPR0EnableVTx() or
1105 * similar was used to enable VT-x on the host.
1106 */
1107VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1108{
1109 NOREF(pCpu);
1110 NOREF(pvCpuPage);
1111 NOREF(HCPhysCpuPage);
1112
1113 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1114 return hmR0VmxLeaveRootMode();
1115}
1116
1117
1118/**
1119 * Sets the permission bits for the specified MSR in the MSR bitmap.
1120 *
1121 * @param pVCpu Pointer to the VMCPU.
1122 * @param uMSR The MSR value.
1123 * @param enmRead Whether reading this MSR causes a VM-exit.
1124 * @param enmWrite Whether writing this MSR causes a VM-exit.
1125 */
1126static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1127{
1128 int32_t iBit;
1129 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1130
1131 /*
1132 * Layout:
1133 * 0x000 - 0x3ff - Low MSR read bits
1134 * 0x400 - 0x7ff - High MSR read bits
1135 * 0x800 - 0xbff - Low MSR write bits
1136 * 0xc00 - 0xfff - High MSR write bits
1137 */
1138 if (uMsr <= 0x00001FFF)
1139 iBit = uMsr;
1140 else if ( uMsr >= 0xC0000000
1141 && uMsr <= 0xC0001FFF)
1142 {
1143 iBit = (uMsr - 0xC0000000);
1144 pbMsrBitmap += 0x400;
1145 }
1146 else
1147 {
1148 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1149 return;
1150 }
1151
1152 Assert(iBit <= 0x1fff);
1153 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154 ASMBitSet(pbMsrBitmap, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap, iBit);
1157
1158 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160 else
1161 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162}
1163
1164
1165#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1166/**
1167 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1168 * area.
1169 *
1170 * @returns VBox status code.
1171 * @param pVCpu Pointer to the VMCPU.
1172 * @param cMsrs The number of MSRs.
1173 */
1174DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1175{
1176 /* Update the VCPU's copy of the guest MSR count. */
1177 pVCpu->hm.s.vmx.cGuestMsrs = cMsrs;
1178
1179 /* Update number of guest MSRs to load/store across the world-switch. */
1180 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1181 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1182
1183 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1184 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1185 return VINF_SUCCESS;
1186}
1187
1188
1189/**
1190 * Adds a guest/host MSR pair to be swapped during the world-switch as
1191 * part of the auto-load/store MSR area in the VMCS.
1192 *
1193 * @returns VBox status code.
1194 * @param pVCpu Pointer to the VMCPU.
1195 * @param uMsr The MSR.
1196 * @param uGuestMsr Value of the guest MSR.
1197 * @param uHostMsr Value of the host MSR.
1198 */
1199static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, uint64_t uHostMsrValue)
1200{
1201 AssertMsg(HMVMX_MAX_SWAP_MSR_COUNT < MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc),
1202 ("MSR swap count exceeded. Cpu reports %#RX32, our limit %#RX32\n",
1203 MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc), HMVMX_MAX_SWAP_MSR_COUNT));
1204
1205 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1206 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1207 uint32_t i;
1208 for (i = 0; i < cGuestMsrs; i++)
1209 {
1210 if (pGuestMsr->u32Msr == uMsr)
1211 break;
1212 pGuestMsr++;
1213 }
1214
1215 AssertReturn(i < HMVMX_MAX_SWAP_MSR_COUNT, VERR_HM_MSR_SWAP_COUNT_EXCEEDED);
1216 if (i == cGuestMsrs)
1217 {
1218 ++cGuestMsrs;
1219 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1220 if (RT_UNLIKELY(cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc)))
1221 {
1222 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
1223 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1224 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1225 }
1226
1227 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1228 AssertRCReturn(rc, rc);
1229 }
1230
1231 /* Update the MSR values in the auto-load/store MSR area. */
1232 pGuestMsr->u32Msr = uMsr;
1233 pGuestMsr->u64Value = uGuestMsrValue;
1234
1235 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1236 pHostMsr += i;
1237 pHostMsr->u32Msr = uMsr;
1238 pHostMsr->u64Value = uHostMsrValue;
1239
1240 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1241 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
1242 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1243 return VINF_SUCCESS;
1244}
1245
1246
1247/**
1248 * Removes a guest/shost MSR pair to be swapped during the world-switch from the
1249 * auto-load/store MSR area in the VMCS.
1250 *
1251 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1252 * area.
1253 *
1254 * @returns VBox status code.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 */
1258static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cGuestMsrs; i++)
1264 {
1265 /* Find the MSR. */
1266 if (pGuestMsr->u32Msr == uMsr)
1267 {
1268 /* If it's the last MSR, simply reduce the count. */
1269 if (i == cGuestMsrs - 1)
1270 {
1271 --cGuestMsrs;
1272 break;
1273 }
1274
1275 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1276 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1277 pLastGuestMsr += cGuestMsrs;
1278 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1279 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1280
1281 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1282 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1283 pLastHostMsr += cGuestMsrs;
1284 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1285 pHostMsr->u64Value = pLastHostMsr->u64Value;
1286 --cGuestMsrs;
1287 break;
1288 }
1289 pGuestMsr++;
1290 }
1291
1292 /* Update the VMCS if the count changed (meaning the MSR was found). */
1293 if (cGuestMsrs != pVCpu->hm.s.vmx.cGuestMsrs)
1294 {
1295 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1296 AssertRCReturn(rc, rc);
1297 }
1298
1299 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1300 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1301 return VINF_SUCCESS;
1302}
1303
1304
1305/**
1306 * Updates the value of a host MSR in the auto-load/store area in the VMCS.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxUpdateAutoLoadStoreHostMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1316
1317 for (uint32_t i = 0; i < cMsrs; i++)
1318 {
1319 if (pHostMsr->u32Msr == uMsr)
1320 {
1321 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1322 return VINF_SUCCESS;
1323 }
1324 }
1325
1326 return VERR_NOT_FOUND;
1327}
1328
1329#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1330
1331
1332/**
1333 * Flushes the TLB using EPT.
1334 *
1335 * @returns VBox status code.
1336 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1337 * enmFlush).
1338 * @param enmFlush Type of flush.
1339 *
1340 * @remarks Caller is responsible for making sure this function is called only
1341 * when NestedPaging is supported and providing @a enmFlush that is
1342 * supported by the CPU.
1343 * @remarks Can be called with interrupts disabled.
1344 */
1345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1346{
1347 uint64_t au64Descriptor[2];
1348 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1349 au64Descriptor[0] = 0;
1350 else
1351 {
1352 Assert(pVCpu);
1353 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1354 }
1355 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1356
1357 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1358 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1359 rc));
1360 if ( RT_SUCCESS(rc)
1361 && pVCpu)
1362 {
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1364 }
1365}
1366
1367
1368/**
1369 * Flushes the TLB using VPID.
1370 *
1371 * @returns VBox status code.
1372 * @param pVM Pointer to the VM.
1373 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1374 * enmFlush).
1375 * @param enmFlush Type of flush.
1376 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1377 * on @a enmFlush).
1378 *
1379 * @remarks Can be called with interrupts disabled.
1380 */
1381static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1382{
1383 AssertPtr(pVM);
1384 Assert(pVM->hm.s.vmx.fVpid);
1385
1386 uint64_t au64Descriptor[2];
1387 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1388 {
1389 au64Descriptor[0] = 0;
1390 au64Descriptor[1] = 0;
1391 }
1392 else
1393 {
1394 AssertPtr(pVCpu);
1395 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1396 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1397 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1398 au64Descriptor[1] = GCPtr;
1399 }
1400
1401 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1402 AssertMsg(rc == VINF_SUCCESS,
1403 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1404 if ( RT_SUCCESS(rc)
1405 && pVCpu)
1406 {
1407 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1408 }
1409}
1410
1411
1412/**
1413 * Invalidates a guest page by guest virtual address. Only relevant for
1414 * EPT/VPID, otherwise there is nothing really to invalidate.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param GCVirt Guest virtual address of the page to invalidate.
1420 */
1421VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1422{
1423 AssertPtr(pVM);
1424 AssertPtr(pVCpu);
1425 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1426
1427 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1428 if (!fFlushPending)
1429 {
1430 /*
1431 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1432 * See @bugref{6043} and @bugref{6177}.
1433 *
1434 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1435 * function maybe called in a loop with individual addresses.
1436 */
1437 if (pVM->hm.s.vmx.fVpid)
1438 {
1439 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1440 {
1441 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1442 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1443 }
1444 else
1445 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1446 }
1447 else if (pVM->hm.s.fNestedPaging)
1448 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1449 }
1450
1451 return VINF_SUCCESS;
1452}
1453
1454
1455/**
1456 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1457 * otherwise there is nothing really to invalidate.
1458 *
1459 * @returns VBox status code.
1460 * @param pVM Pointer to the VM.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param GCPhys Guest physical address of the page to invalidate.
1463 */
1464VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1465{
1466 LogFlowFunc(("%RGp\n", GCPhys));
1467
1468 /*
1469 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1470 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1471 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1472 */
1473 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1474 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1475 return VINF_SUCCESS;
1476}
1477
1478
1479/**
1480 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1481 * case where neither EPT nor VPID is supported by the CPU.
1482 *
1483 * @param pVM Pointer to the VM.
1484 * @param pVCpu Pointer to the VMCPU.
1485 * @param pCpu Pointer to the global HM struct.
1486 *
1487 * @remarks Called with interrupts disabled.
1488 */
1489static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1490{
1491 AssertPtr(pVCpu);
1492 AssertPtr(pCpu);
1493 NOREF(pVM);
1494
1495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1496
1497 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1498#if 0
1499 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1500 pVCpu->hm.s.TlbShootdown.cPages = 0;
1501#endif
1502
1503 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1504 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1505 pVCpu->hm.s.fForceTLBFlush = false;
1506 return;
1507}
1508
1509
1510/**
1511 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1512 *
1513 * @param pVM Pointer to the VM.
1514 * @param pVCpu Pointer to the VMCPU.
1515 * @param pCpu Pointer to the global HM CPU struct.
1516 * @remarks All references to "ASID" in this function pertains to "VPID" in
1517 * Intel's nomenclature. The reason is, to avoid confusion in compare
1518 * statements since the host-CPU copies are named "ASID".
1519 *
1520 * @remarks Called with interrupts disabled.
1521 */
1522static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1523{
1524#ifdef VBOX_WITH_STATISTICS
1525 bool fTlbFlushed = false;
1526# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1527# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1528 if (!fTlbFlushed) \
1529 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1530 } while (0)
1531#else
1532# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1533# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1534#endif
1535
1536 AssertPtr(pVM);
1537 AssertPtr(pCpu);
1538 AssertPtr(pVCpu);
1539 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1540 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1541 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1542
1543
1544 /*
1545 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1546 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1547 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1548 */
1549 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1550 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1551 {
1552 ++pCpu->uCurrentAsid;
1553 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1554 {
1555 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1556 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1557 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1558 }
1559
1560 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1561 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1562 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1563
1564 /*
1565 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1566 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1567 */
1568 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1569 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1570 HMVMX_SET_TAGGED_TLB_FLUSHED();
1571 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1572 }
1573
1574 /* Check for explicit TLB shootdowns. */
1575 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1576 {
1577 /*
1578 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1579 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1580 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1581 * but not guest-physical mappings.
1582 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1583 */
1584 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1585 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1586 HMVMX_SET_TAGGED_TLB_FLUSHED();
1587 }
1588
1589 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1590 * where it is commented out. Support individual entry flushing
1591 * someday. */
1592#if 0
1593 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1594 {
1595 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1596
1597 /*
1598 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1599 * as supported by the CPU.
1600 */
1601 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1602 {
1603 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1604 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1605 }
1606 else
1607 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1608
1609 HMVMX_SET_TAGGED_TLB_FLUSHED();
1610 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1611 pVCpu->hm.s.TlbShootdown.cPages = 0;
1612 }
1613#endif
1614
1615 pVCpu->hm.s.fForceTLBFlush = false;
1616
1617 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1618
1619 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1620 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1621 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1622 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1623 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1624 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1625 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1626 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1627
1628 /* Update VMCS with the VPID. */
1629 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1630 AssertRC(rc);
1631
1632#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1633}
1634
1635
1636/**
1637 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1638 *
1639 * @returns VBox status code.
1640 * @param pVM Pointer to the VM.
1641 * @param pVCpu Pointer to the VMCPU.
1642 * @param pCpu Pointer to the global HM CPU struct.
1643 *
1644 * @remarks Called with interrupts disabled.
1645 */
1646static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1647{
1648 AssertPtr(pVM);
1649 AssertPtr(pVCpu);
1650 AssertPtr(pCpu);
1651 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1652 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1653
1654 /*
1655 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1656 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1657 */
1658 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1659 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1660 {
1661 pVCpu->hm.s.fForceTLBFlush = true;
1662 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1663 }
1664
1665 /* Check for explicit TLB shootdown flushes. */
1666 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1667 {
1668 pVCpu->hm.s.fForceTLBFlush = true;
1669 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1670 }
1671
1672 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1673 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1674
1675 if (pVCpu->hm.s.fForceTLBFlush)
1676 {
1677 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1678 pVCpu->hm.s.fForceTLBFlush = false;
1679 }
1680 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1681 * where it is commented out. Support individual entry flushing
1682 * someday. */
1683#if 0
1684 else
1685 {
1686 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1687 {
1688 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1689 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1690 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1691 }
1692 else
1693 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1694
1695 pVCpu->hm.s.TlbShootdown.cPages = 0;
1696 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1697 }
1698#endif
1699}
1700
1701
1702/**
1703 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1704 *
1705 * @returns VBox status code.
1706 * @param pVM Pointer to the VM.
1707 * @param pVCpu Pointer to the VMCPU.
1708 * @param pCpu Pointer to the global HM CPU struct.
1709 *
1710 * @remarks Called with interrupts disabled.
1711 */
1712static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1713{
1714 AssertPtr(pVM);
1715 AssertPtr(pVCpu);
1716 AssertPtr(pCpu);
1717 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1718 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1719
1720 /*
1721 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1722 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1723 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1724 */
1725 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1726 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1727 {
1728 pVCpu->hm.s.fForceTLBFlush = true;
1729 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1730 }
1731
1732 /* Check for explicit TLB shootdown flushes. */
1733 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1734 {
1735 /*
1736 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1737 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1738 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1739 */
1740 pVCpu->hm.s.fForceTLBFlush = true;
1741 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1742 }
1743
1744 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1745 if (pVCpu->hm.s.fForceTLBFlush)
1746 {
1747 ++pCpu->uCurrentAsid;
1748 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1749 {
1750 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1751 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1752 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1753 }
1754
1755 pVCpu->hm.s.fForceTLBFlush = false;
1756 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1757 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1758 if (pCpu->fFlushAsidBeforeUse)
1759 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1760 }
1761 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1762 * where it is commented out. Support individual entry flushing
1763 * someday. */
1764#if 0
1765 else
1766 {
1767 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1768 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1769 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1770 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1771
1772 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1773 {
1774 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1776 {
1777 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1778 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1779 }
1780 else
1781 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1782
1783 pVCpu->hm.s.TlbShootdown.cPages = 0;
1784 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1785 }
1786 else
1787 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1788 }
1789#endif
1790
1791 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1792 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1793 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1794 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1795 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1796 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1797
1798 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1799 AssertRC(rc);
1800}
1801
1802
1803/**
1804 * Flushes the guest TLB entry based on CPU capabilities.
1805 *
1806 * @param pVCpu Pointer to the VMCPU.
1807 * @param pCpu Pointer to the global HM CPU struct.
1808 */
1809DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1810{
1811#ifdef HMVMX_ALWAYS_FLUSH_TLB
1812 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1813#endif
1814 PVM pVM = pVCpu->CTX_SUFF(pVM);
1815 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1816 {
1817 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1818 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1819 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1820 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1821 default:
1822 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1823 break;
1824 }
1825
1826 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
1827 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
1828
1829 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
1830}
1831
1832
1833/**
1834 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1835 * TLB entries from the host TLB before VM-entry.
1836 *
1837 * @returns VBox status code.
1838 * @param pVM Pointer to the VM.
1839 */
1840static int hmR0VmxSetupTaggedTlb(PVM pVM)
1841{
1842 /*
1843 * Determine optimal flush type for Nested Paging.
1844 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1845 * guest execution (see hmR3InitFinalizeR0()).
1846 */
1847 if (pVM->hm.s.fNestedPaging)
1848 {
1849 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1850 {
1851 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1852 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1853 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1854 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1855 else
1856 {
1857 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1858 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1859 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1860 }
1861
1862 /* Make sure the write-back cacheable memory type for EPT is supported. */
1863 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1864 {
1865 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1866 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1867 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1868 }
1869 }
1870 else
1871 {
1872 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1873 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1874 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1875 }
1876 }
1877
1878 /*
1879 * Determine optimal flush type for VPID.
1880 */
1881 if (pVM->hm.s.vmx.fVpid)
1882 {
1883 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1884 {
1885 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1886 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1887 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1888 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1889 else
1890 {
1891 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1892 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1893 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1894 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1895 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1896 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1897 pVM->hm.s.vmx.fVpid = false;
1898 }
1899 }
1900 else
1901 {
1902 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1903 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1904 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1905 pVM->hm.s.vmx.fVpid = false;
1906 }
1907 }
1908
1909 /*
1910 * Setup the handler for flushing tagged-TLBs.
1911 */
1912 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1913 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1914 else if (pVM->hm.s.fNestedPaging)
1915 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1916 else if (pVM->hm.s.vmx.fVpid)
1917 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1918 else
1919 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1920 return VINF_SUCCESS;
1921}
1922
1923
1924/**
1925 * Sets up pin-based VM-execution controls in the VMCS.
1926 *
1927 * @returns VBox status code.
1928 * @param pVM Pointer to the VM.
1929 * @param pVCpu Pointer to the VMCPU.
1930 */
1931static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1932{
1933 AssertPtr(pVM);
1934 AssertPtr(pVCpu);
1935
1936 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1937 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1938
1939 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1940 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1941 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1942
1943 /* Enable the VMX preemption timer. */
1944 if (pVM->hm.s.vmx.fUsePreemptTimer)
1945 {
1946 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1947 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1948 }
1949
1950 if ((val & zap) != val)
1951 {
1952 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1953 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1954 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1955 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1956 }
1957
1958 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1959 AssertRCReturn(rc, rc);
1960
1961 /* Update VCPU with the currently set pin-based VM-execution controls. */
1962 pVCpu->hm.s.vmx.u32PinCtls = val;
1963 return rc;
1964}
1965
1966
1967/**
1968 * Sets up processor-based VM-execution controls in the VMCS.
1969 *
1970 * @returns VBox status code.
1971 * @param pVM Pointer to the VM.
1972 * @param pVMCPU Pointer to the VMCPU.
1973 */
1974static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1975{
1976 AssertPtr(pVM);
1977 AssertPtr(pVCpu);
1978
1979 int rc = VERR_INTERNAL_ERROR_5;
1980 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1981 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1982
1983 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1984 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1985 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1986 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1987 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1988 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1989 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1990
1991 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1992 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1993 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1994 {
1995 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1996 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1997 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1998 }
1999
2000 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2001 if (!pVM->hm.s.fNestedPaging)
2002 {
2003 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2004 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2005 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2006 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2007 }
2008
2009 /* Use TPR shadowing if supported by the CPU. */
2010 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2011 {
2012 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2013 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2014 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2015 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2016 AssertRCReturn(rc, rc);
2017
2018 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2019 /* CR8 writes causes a VM-exit based on TPR threshold. */
2020 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2021 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2022 }
2023 else
2024 {
2025 /*
2026 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2027 * Set this control only for 64-bit guests.
2028 */
2029 if (pVM->hm.s.fAllow64BitGuests)
2030 {
2031 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2032 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2033 }
2034 }
2035
2036 /* Use MSR-bitmaps if supported by the CPU. */
2037 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2038 {
2039 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2040
2041 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2042 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2043 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2044 AssertRCReturn(rc, rc);
2045
2046 /*
2047 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2048 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
2049 */
2050 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2051 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2052 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2053 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2054 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2055 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2056 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2057 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2058 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2059 }
2060
2061 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2062 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2063 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2064
2065 if ((val & zap) != val)
2066 {
2067 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2068 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2069 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2070 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2071 }
2072
2073 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2074 AssertRCReturn(rc, rc);
2075
2076 /* Update VCPU with the currently set processor-based VM-execution controls. */
2077 pVCpu->hm.s.vmx.u32ProcCtls = val;
2078
2079 /*
2080 * Secondary processor-based VM-execution controls.
2081 */
2082 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2083 {
2084 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2085 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2086
2087 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2088 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2089
2090 if (pVM->hm.s.fNestedPaging)
2091 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2092 else
2093 {
2094 /*
2095 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2096 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2097 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2098 */
2099 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2100 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2101 }
2102
2103 if (pVM->hm.s.vmx.fVpid)
2104 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2105
2106 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2107 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2108
2109 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2110 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2111 * done dynamically. */
2112 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2113 {
2114 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2115 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2116 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2117 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2118 AssertRCReturn(rc, rc);
2119 }
2120
2121 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2122 {
2123 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2124 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2125 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2126 }
2127
2128 if ((val & zap) != val)
2129 {
2130 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2131 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2132 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2133 }
2134
2135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2136 AssertRCReturn(rc, rc);
2137
2138 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2139 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2140 }
2141 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2142 {
2143 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2144 "available\n"));
2145 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2146 }
2147
2148 return VINF_SUCCESS;
2149}
2150
2151
2152/**
2153 * Sets up miscellaneous (everything other than Pin & Processor-based
2154 * VM-execution) control fields in the VMCS.
2155 *
2156 * @returns VBox status code.
2157 * @param pVM Pointer to the VM.
2158 * @param pVCpu Pointer to the VMCPU.
2159 */
2160static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2161{
2162 AssertPtr(pVM);
2163 AssertPtr(pVCpu);
2164
2165 int rc = VERR_GENERAL_FAILURE;
2166
2167 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2168#if 0
2169 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2170 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2171 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2172
2173 /*
2174 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2175 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2176 * We thus use the exception bitmap to control it rather than use both.
2177 */
2178 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2179 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2180
2181 /** @todo Explore possibility of using IO-bitmaps. */
2182 /* All IO & IOIO instructions cause VM-exits. */
2183 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2184 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2185
2186 /* Initialize the MSR-bitmap area. */
2187 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2188 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2189 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2190#endif
2191
2192#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2193 /* Setup MSR autoloading/storing. */
2194 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2195 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2196 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2197 AssertRCReturn(rc, rc);
2198 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2199 AssertRCReturn(rc, rc);
2200
2201 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2202 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2203 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2204 AssertRCReturn(rc, rc);
2205#endif
2206
2207 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2208 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2209 AssertRCReturn(rc, rc);
2210
2211 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2212#if 0
2213 /* Setup debug controls */
2214 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2215 AssertRCReturn(rc, rc);
2216 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2217 AssertRCReturn(rc, rc);
2218#endif
2219
2220 return rc;
2221}
2222
2223
2224/**
2225 * Sets up the initial exception bitmap in the VMCS based on static conditions
2226 * (i.e. conditions that cannot ever change after starting the VM).
2227 *
2228 * @returns VBox status code.
2229 * @param pVM Pointer to the VM.
2230 * @param pVCpu Pointer to the VMCPU.
2231 */
2232static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2233{
2234 AssertPtr(pVM);
2235 AssertPtr(pVCpu);
2236
2237 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2238
2239 uint32_t u32XcptBitmap = 0;
2240
2241 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2242 if (!pVM->hm.s.fNestedPaging)
2243 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2244
2245 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2246 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2247 AssertRCReturn(rc, rc);
2248 return rc;
2249}
2250
2251
2252/**
2253 * Sets up the initial guest-state mask. The guest-state mask is consulted
2254 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2255 * for the nested virtualization case (as it would cause a VM-exit).
2256 *
2257 * @param pVCpu Pointer to the VMCPU.
2258 */
2259static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2260{
2261 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2262 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Does per-VM VT-x initialization.
2269 *
2270 * @returns VBox status code.
2271 * @param pVM Pointer to the VM.
2272 */
2273VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2274{
2275 LogFlowFunc(("pVM=%p\n", pVM));
2276
2277 int rc = hmR0VmxStructsAlloc(pVM);
2278 if (RT_FAILURE(rc))
2279 {
2280 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2281 return rc;
2282 }
2283
2284 return VINF_SUCCESS;
2285}
2286
2287
2288/**
2289 * Does per-VM VT-x termination.
2290 *
2291 * @returns VBox status code.
2292 * @param pVM Pointer to the VM.
2293 */
2294VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2295{
2296 LogFlowFunc(("pVM=%p\n", pVM));
2297
2298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2299 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2300 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2301#endif
2302 hmR0VmxStructsFree(pVM);
2303 return VINF_SUCCESS;
2304}
2305
2306
2307/**
2308 * Sets up the VM for execution under VT-x.
2309 * This function is only called once per-VM during initialization.
2310 *
2311 * @returns VBox status code.
2312 * @param pVM Pointer to the VM.
2313 */
2314VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2315{
2316 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2317 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2318
2319 LogFlowFunc(("pVM=%p\n", pVM));
2320
2321 /*
2322 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2323 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2324 */
2325 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2326 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2327 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2328 || !pVM->hm.s.vmx.pRealModeTSS))
2329 {
2330 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2331 return VERR_INTERNAL_ERROR;
2332 }
2333
2334#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2335 /*
2336 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2337 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2338 */
2339 if ( pVM->hm.s.fAllow64BitGuests
2340 && !HMVMX_IS_64BIT_HOST_MODE())
2341 {
2342 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2343 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2344 }
2345#endif
2346
2347 /* Initialize these always, see hmR3InitFinalizeR0().*/
2348 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2349 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2350
2351 /* Setup the tagged-TLB flush handlers. */
2352 int rc = hmR0VmxSetupTaggedTlb(pVM);
2353 if (RT_FAILURE(rc))
2354 {
2355 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2356 return rc;
2357 }
2358
2359 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2360 {
2361 PVMCPU pVCpu = &pVM->aCpus[i];
2362 AssertPtr(pVCpu);
2363 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2364
2365 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2366 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2367
2368 /* Set revision dword at the beginning of the VMCS structure. */
2369 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2370
2371 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2372 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2373 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2374 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2375
2376 /* Load this VMCS as the current VMCS. */
2377 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2378 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2379 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2380
2381 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2382 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2383 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2384
2385 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2386 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2387 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2388
2389 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2390 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2391 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2392
2393 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2394 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2395 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2396
2397 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2398 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2399 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2400
2401#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2402 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2403 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2404 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2405#endif
2406
2407 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2408 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2409 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2410 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2411
2412 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2413
2414 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2415 }
2416
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2423 * the VMCS.
2424 *
2425 * @returns VBox status code.
2426 * @param pVM Pointer to the VM.
2427 * @param pVCpu Pointer to the VMCPU.
2428 */
2429DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2430{
2431 RTCCUINTREG uReg = ASMGetCR0();
2432 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2433 AssertRCReturn(rc, rc);
2434
2435#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2436 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2437 if (HMVMX_IS_64BIT_HOST_MODE())
2438 {
2439 uint64_t uRegCR3 = HMR0Get64bitCR3();
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2441 }
2442 else
2443#endif
2444 {
2445 uReg = ASMGetCR3();
2446 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2447 }
2448 AssertRCReturn(rc, rc);
2449
2450 uReg = ASMGetCR4();
2451 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2452 AssertRCReturn(rc, rc);
2453 return rc;
2454}
2455
2456
2457#if HC_ARCH_BITS == 64
2458/**
2459 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2460 * requirements. See hmR0VmxSaveHostSegmentRegs().
2461 */
2462# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2463 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2464 { \
2465 bool fValidSelector = true; \
2466 if ((selValue) & X86_SEL_LDT) \
2467 { \
2468 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2469 fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2470 } \
2471 if (fValidSelector) \
2472 { \
2473 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2474 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2475 } \
2476 (selValue) = 0; \
2477 }
2478#endif
2479
2480
2481/**
2482 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2483 * the host-state area in the VMCS.
2484 *
2485 * @returns VBox status code.
2486 * @param pVM Pointer to the VM.
2487 * @param pVCpu Pointer to the VMCPU.
2488 */
2489DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2490{
2491 int rc = VERR_INTERNAL_ERROR_5;
2492
2493 /*
2494 * Host DS, ES, FS and GS segment registers.
2495 */
2496#if HC_ARCH_BITS == 64
2497 RTSEL uSelDS = ASMGetDS();
2498 RTSEL uSelES = ASMGetES();
2499 RTSEL uSelFS = ASMGetFS();
2500 RTSEL uSelGS = ASMGetGS();
2501#else
2502 RTSEL uSelDS = 0;
2503 RTSEL uSelES = 0;
2504 RTSEL uSelFS = 0;
2505 RTSEL uSelGS = 0;
2506#endif
2507
2508 /* Recalculate which host-state bits need to be manually restored. */
2509 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2510
2511 /*
2512 * Host CS and SS segment registers.
2513 */
2514#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2515 RTSEL uSelCS;
2516 RTSEL uSelSS;
2517 if (HMVMX_IS_64BIT_HOST_MODE())
2518 {
2519 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2520 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2521 }
2522 else
2523 {
2524 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2525 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2526 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2527 }
2528#else
2529 RTSEL uSelCS = ASMGetCS();
2530 RTSEL uSelSS = ASMGetSS();
2531#endif
2532
2533 /*
2534 * Host TR segment register.
2535 */
2536 RTSEL uSelTR = ASMGetTR();
2537
2538#if HC_ARCH_BITS == 64
2539 /*
2540 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2541 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2542 */
2543 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2544 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2545 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2546 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2547# undef VMXLOCAL_ADJUST_HOST_SEG
2548#endif
2549
2550 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2551 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2552 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2553 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2554 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2555 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2556 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2557 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2558 Assert(uSelCS);
2559 Assert(uSelTR);
2560
2561 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2562#if 0
2563 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2564 Assert(uSelSS != 0);
2565#endif
2566
2567 /* Write these host selector fields into the host-state area in the VMCS. */
2568 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2570#if HC_ARCH_BITS == 64
2571 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2575#endif
2576 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2577
2578 /*
2579 * Host GDTR and IDTR.
2580 */
2581 RTGDTR Gdtr;
2582 RT_ZERO(Gdtr);
2583#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2584 if (HMVMX_IS_64BIT_HOST_MODE())
2585 {
2586 X86XDTR64 Gdtr64;
2587 X86XDTR64 Idtr64;
2588 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2589 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2591
2592 Gdtr.cbGdt = Gdtr64.cb;
2593 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2594 }
2595 else
2596#endif
2597 {
2598 RTIDTR Idtr;
2599 ASMGetGDTR(&Gdtr);
2600 ASMGetIDTR(&Idtr);
2601 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2602 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2603
2604#if HC_ARCH_BITS == 64
2605 /*
2606 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2607 * maximum limit (0xffff) on every VM-exit.
2608 */
2609 if (Gdtr.cbGdt != 0xffff)
2610 {
2611 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2612 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2613 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2614 }
2615
2616 /*
2617 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2618 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2619 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2620 */
2621 if (Idtr.cbIdt < 0x0fff)
2622 {
2623 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2624 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2625 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2626 }
2627#endif
2628 }
2629
2630 /*
2631 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2632 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2633 */
2634 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2635 {
2636 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2637 return VERR_VMX_INVALID_HOST_STATE;
2638 }
2639
2640 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2641#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2642 if (HMVMX_IS_64BIT_HOST_MODE())
2643 {
2644 /* We need the 64-bit TR base for hybrid darwin. */
2645 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2646 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2647 }
2648 else
2649#endif
2650 {
2651 uintptr_t uTRBase;
2652#if HC_ARCH_BITS == 64
2653 uTRBase = X86DESC64_BASE(pDesc);
2654
2655 /*
2656 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2657 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2658 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2659 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2660 *
2661 * [1] See Intel spec. 3.5 "System Descriptor Types".
2662 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2663 */
2664 Assert(pDesc->System.u4Type == 11);
2665 if ( pDesc->System.u16LimitLow != 0x67
2666 || pDesc->System.u4LimitHigh)
2667 {
2668 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2669 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2670
2671 /* Store the GDTR here as we need it while restoring TR. */
2672 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2673 }
2674#else
2675 uTRBase = X86DESC_BASE(pDesc);
2676#endif
2677 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2678 }
2679 AssertRCReturn(rc, rc);
2680
2681 /*
2682 * Host FS base and GS base.
2683 */
2684#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2685 if (HMVMX_IS_64BIT_HOST_MODE())
2686 {
2687 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2688 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2689 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2690 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2691
2692# if HC_ARCH_BITS == 64
2693 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2694 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2695 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2696 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2697 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2698# endif
2699 }
2700#endif
2701 return rc;
2702}
2703
2704
2705/**
2706 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2707 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2708 * the host after every successful VM exit.
2709 *
2710 * @returns VBox status code.
2711 * @param pVM Pointer to the VM.
2712 * @param pVCpu Pointer to the VMCPU.
2713 */
2714DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2715{
2716 AssertPtr(pVCpu);
2717 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2718
2719 int rc = VINF_SUCCESS;
2720#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2721 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2722 uint32_t cHostMsrs = 0;
2723 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2724
2725 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2726 {
2727 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2728
2729# if HC_ARCH_BITS == 64
2730 /* Paranoia. 64-bit code requires these bits to be set always. */
2731 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2732
2733 /*
2734 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2735 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2736 * some reason (e.g. allow transparent reads) we would activate the code below.
2737 */
2738# if 0
2739 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2740 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2741 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2742 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2743 if (CPUMIsGuestInLongMode(pVCpu))
2744 {
2745 uint64_t u64GuestEfer;
2746 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2747 AssertRC(rc);
2748
2749 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2750 {
2751 pHostMsr->u32Msr = MSR_K6_EFER;
2752 pHostMsr->u32Reserved = 0;
2753 pHostMsr->u64Value = u64HostEfer;
2754 pHostMsr++; cHostMsrs++;
2755 }
2756 }
2757# endif
2758# else /* HC_ARCH_BITS != 64 */
2759 pHostMsr->u32Msr = MSR_K6_EFER;
2760 pHostMsr->u32Reserved = 0;
2761# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2762 if (CPUMIsGuestInLongMode(pVCpu))
2763 {
2764 /* Must match the EFER value in our 64 bits switcher. */
2765 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2766 }
2767 else
2768# endif
2769 pHostMsr->u64Value = u64HostEfer;
2770 pHostMsr++; cHostMsrs++;
2771# endif /* HC_ARCH_BITS == 64 */
2772 }
2773
2774# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2775 if (HMVMX_IS_64BIT_HOST_MODE())
2776 {
2777 pHostMsr->u32Msr = MSR_K6_STAR;
2778 pHostMsr->u32Reserved = 0;
2779 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2780 pHostMsr++; cHostMsrs++;
2781 pHostMsr->u32Msr = MSR_K8_LSTAR;
2782 pHostMsr->u32Reserved = 0;
2783 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2784 pHostMsr++; cHostMsrs++;
2785 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2786 pHostMsr->u32Reserved = 0;
2787 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2788 pHostMsr++; cHostMsrs++;
2789 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2790 pHostMsr->u32Reserved = 0;
2791 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2792 pHostMsr++; cHostMsrs++;
2793 }
2794# endif
2795
2796 /* Host TSC AUX MSR must be restored since we always load/store guest TSC AUX MSR. */
2797 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2798 {
2799 pHostMsr->u32Msr = MSR_K8_TSC_AUX;
2800 pHostMsr->u32Reserved = 0;
2801 pHostMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
2802 pHostMsr++; cHostMsrs++;
2803 }
2804
2805 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2806 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2807 {
2808 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2809 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2810 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2811 }
2812
2813 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2814#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2815
2816 /*
2817 * Host Sysenter MSRs.
2818 */
2819 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2820 AssertRCReturn(rc, rc);
2821#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2822 if (HMVMX_IS_64BIT_HOST_MODE())
2823 {
2824 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2825 AssertRCReturn(rc, rc);
2826 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2827 }
2828 else
2829 {
2830 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2831 AssertRCReturn(rc, rc);
2832 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2833 }
2834#elif HC_ARCH_BITS == 32
2835 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2836 AssertRCReturn(rc, rc);
2837 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2838#else
2839 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2840 AssertRCReturn(rc, rc);
2841 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2842#endif
2843 AssertRCReturn(rc, rc);
2844
2845 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2846 * hmR0VmxSetupExitCtls() !! */
2847 return rc;
2848}
2849
2850
2851/**
2852 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2853 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2854 * controls".
2855 *
2856 * @returns VBox status code.
2857 * @param pVCpu Pointer to the VMCPU.
2858 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2859 * out-of-sync. Make sure to update the required fields
2860 * before using them.
2861 *
2862 * @remarks No-long-jump zone!!!
2863 */
2864DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2865{
2866 int rc = VINF_SUCCESS;
2867 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2868 {
2869 PVM pVM = pVCpu->CTX_SUFF(pVM);
2870 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2871 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2872
2873 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2874 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2875
2876 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2877 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2878 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2879 else
2880 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2881
2882 /*
2883 * The following should -not- be set (since we're not in SMM mode):
2884 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2885 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2886 */
2887
2888 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2889 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2890 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2891
2892 if ((val & zap) != val)
2893 {
2894 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2895 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2896 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2897 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2898 }
2899
2900 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2901 AssertRCReturn(rc, rc);
2902
2903 /* Update VCPU with the currently set VM-exit controls. */
2904 pVCpu->hm.s.vmx.u32EntryCtls = val;
2905 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2906 }
2907 return rc;
2908}
2909
2910
2911/**
2912 * Sets up the VM-exit controls in the VMCS.
2913 *
2914 * @returns VBox status code.
2915 * @param pVM Pointer to the VM.
2916 * @param pVCpu Pointer to the VMCPU.
2917 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2918 * out-of-sync. Make sure to update the required fields
2919 * before using them.
2920 *
2921 * @remarks requires EFER.
2922 */
2923DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2924{
2925 int rc = VINF_SUCCESS;
2926 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2927 {
2928 PVM pVM = pVCpu->CTX_SUFF(pVM);
2929 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2930 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2931
2932 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2933 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2934
2935 /*
2936 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2937 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2938 */
2939#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2940 if (HMVMX_IS_64BIT_HOST_MODE())
2941 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2942 else
2943 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2944#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2945 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2946 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2947 else
2948 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2949#endif
2950
2951 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2952 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2953
2954 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2955 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2956 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2957 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2958 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2959
2960 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2961 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2962
2963 if ((val & zap) != val)
2964 {
2965 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2966 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2967 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2968 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2969 }
2970
2971 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2972 AssertRCReturn(rc, rc);
2973
2974 /* Update VCPU with the currently set VM-exit controls. */
2975 pVCpu->hm.s.vmx.u32ExitCtls = val;
2976 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2977 }
2978 return rc;
2979}
2980
2981
2982/**
2983 * Loads the guest APIC and related state.
2984 *
2985 * @returns VBox status code.
2986 * @param pVM Pointer to the VM.
2987 * @param pVCpu Pointer to the VMCPU.
2988 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2989 * out-of-sync. Make sure to update the required fields
2990 * before using them.
2991 */
2992DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2993{
2994 int rc = VINF_SUCCESS;
2995 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2996 {
2997 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2998 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2999 {
3000 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3001
3002 bool fPendingIntr = false;
3003 uint8_t u8Tpr = 0;
3004 uint8_t u8PendingIntr = 0;
3005 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3006 AssertRCReturn(rc, rc);
3007
3008 /*
3009 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3010 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3011 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3012 * the interrupt when we VM-exit for other reasons.
3013 */
3014 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3015 uint32_t u32TprThreshold = 0;
3016 if (fPendingIntr)
3017 {
3018 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3019 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3020 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3021 if (u8PendingPriority <= u8TprPriority)
3022 u32TprThreshold = u8PendingPriority;
3023 else
3024 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3025 }
3026 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3027
3028 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3029 AssertRCReturn(rc, rc);
3030 }
3031
3032 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3033 }
3034 return rc;
3035}
3036
3037
3038/**
3039 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3040 *
3041 * @returns Guest's interruptibility-state.
3042 * @param pVCpu Pointer to the VMCPU.
3043 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3044 * out-of-sync. Make sure to update the required fields
3045 * before using them.
3046 *
3047 * @remarks No-long-jump zone!!!
3048 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3049 */
3050DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3051{
3052 /*
3053 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3054 * inhibit interrupts or clear any existing interrupt-inhibition.
3055 */
3056 uint32_t uIntrState = 0;
3057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3058 {
3059 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3060 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
3061 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3062 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3063 {
3064 /*
3065 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3066 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3067 */
3068 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3069 }
3070 else if (pMixedCtx->eflags.Bits.u1IF)
3071 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3072 else
3073 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3074 }
3075 return uIntrState;
3076}
3077
3078
3079/**
3080 * Loads the guest's interruptibility-state into the guest-state area in the
3081 * VMCS.
3082 *
3083 * @returns VBox status code.
3084 * @param pVCpu Pointer to the VMCPU.
3085 * @param uIntrState The interruptibility-state to set.
3086 */
3087static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3088{
3089 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3090 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3091 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3092 AssertRCReturn(rc, rc);
3093 return rc;
3094}
3095
3096
3097/**
3098 * Loads the guest's RIP into the guest-state area in the VMCS.
3099 *
3100 * @returns VBox status code.
3101 * @param pVCpu Pointer to the VMCPU.
3102 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3103 * out-of-sync. Make sure to update the required fields
3104 * before using them.
3105 *
3106 * @remarks No-long-jump zone!!!
3107 */
3108static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3109{
3110 int rc = VINF_SUCCESS;
3111 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3112 {
3113 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3114 AssertRCReturn(rc, rc);
3115
3116 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3117 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3118 }
3119 return rc;
3120}
3121
3122
3123/**
3124 * Loads the guest's RSP into the guest-state area in the VMCS.
3125 *
3126 * @returns VBox status code.
3127 * @param pVCpu Pointer to the VMCPU.
3128 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3129 * out-of-sync. Make sure to update the required fields
3130 * before using them.
3131 *
3132 * @remarks No-long-jump zone!!!
3133 */
3134static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3135{
3136 int rc = VINF_SUCCESS;
3137 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3138 {
3139 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3140 AssertRCReturn(rc, rc);
3141
3142 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3143 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3144 }
3145 return rc;
3146}
3147
3148
3149/**
3150 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3151 *
3152 * @returns VBox status code.
3153 * @param pVCpu Pointer to the VMCPU.
3154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3155 * out-of-sync. Make sure to update the required fields
3156 * before using them.
3157 *
3158 * @remarks No-long-jump zone!!!
3159 */
3160static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3161{
3162 int rc = VINF_SUCCESS;
3163 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3164 {
3165 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3166 Let us assert it as such and use 32-bit VMWRITE. */
3167 Assert(!(pMixedCtx->rflags.u64 >> 32));
3168 X86EFLAGS Eflags = pMixedCtx->eflags;
3169 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3170 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3171
3172 /*
3173 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3174 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3175 */
3176 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3177 {
3178 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3179 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3180 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3181 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3182 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3183 }
3184
3185 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3186 AssertRCReturn(rc, rc);
3187
3188 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3189 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3190 }
3191 return rc;
3192}
3193
3194
3195/**
3196 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3197 *
3198 * @returns VBox status code.
3199 * @param pVCpu Pointer to the VMCPU.
3200 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201 * out-of-sync. Make sure to update the required fields
3202 * before using them.
3203 *
3204 * @remarks No-long-jump zone!!!
3205 */
3206DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3207{
3208 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3209 AssertRCReturn(rc, rc);
3210 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3211 AssertRCReturn(rc, rc);
3212 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3213 AssertRCReturn(rc, rc);
3214 return rc;
3215}
3216
3217
3218/**
3219 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3220 * CR0 is partially shared with the host and we have to consider the FPU bits.
3221 *
3222 * @returns VBox status code.
3223 * @param pVM Pointer to the VM.
3224 * @param pVCpu Pointer to the VMCPU.
3225 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3226 * out-of-sync. Make sure to update the required fields
3227 * before using them.
3228 *
3229 * @remarks No-long-jump zone!!!
3230 */
3231static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3232{
3233 /*
3234 * Guest CR0.
3235 * Guest FPU.
3236 */
3237 int rc = VINF_SUCCESS;
3238 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3239 {
3240 Assert(!(pMixedCtx->cr0 >> 32));
3241 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3242 PVM pVM = pVCpu->CTX_SUFF(pVM);
3243
3244 /* The guest's view (read access) of its CR0 is unblemished. */
3245 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3246 AssertRCReturn(rc, rc);
3247 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3248
3249 /* Setup VT-x's view of the guest CR0. */
3250 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3251 if (pVM->hm.s.fNestedPaging)
3252 {
3253 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3254 {
3255 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3256 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3257 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3258 }
3259 else
3260 {
3261 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3262 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3263 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3264 }
3265
3266 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3267 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3268 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3269
3270 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3271 AssertRCReturn(rc, rc);
3272 }
3273 else
3274 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3275
3276 /*
3277 * Guest FPU bits.
3278 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3279 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3280 */
3281 u32GuestCR0 |= X86_CR0_NE;
3282 bool fInterceptNM = false;
3283 if (CPUMIsGuestFPUStateActive(pVCpu))
3284 {
3285 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3286 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3287 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3288 }
3289 else
3290 {
3291 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3292 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3293 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3294 }
3295
3296 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3297 bool fInterceptMF = false;
3298 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3299 fInterceptMF = true;
3300
3301 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3302 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3303 {
3304 Assert(PDMVmmDevHeapIsEnabled(pVM));
3305 Assert(pVM->hm.s.vmx.pRealModeTSS);
3306 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3307 fInterceptNM = true;
3308 fInterceptMF = true;
3309 }
3310 else
3311 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3312
3313 if (fInterceptNM)
3314 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3315 else
3316 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3317
3318 if (fInterceptMF)
3319 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3320 else
3321 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3322
3323 /* Additional intercepts for debugging, define these yourself explicitly. */
3324#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3325 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3326 | RT_BIT(X86_XCPT_BP)
3327 | RT_BIT(X86_XCPT_DB)
3328 | RT_BIT(X86_XCPT_DE)
3329 | RT_BIT(X86_XCPT_NM)
3330 | RT_BIT(X86_XCPT_UD)
3331 | RT_BIT(X86_XCPT_NP)
3332 | RT_BIT(X86_XCPT_SS)
3333 | RT_BIT(X86_XCPT_GP)
3334 | RT_BIT(X86_XCPT_PF)
3335 | RT_BIT(X86_XCPT_MF)
3336 ;
3337#elif defined(HMVMX_ALWAYS_TRAP_PF)
3338 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3339#endif
3340
3341 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3342
3343 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3344 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3345 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3346 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3347 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3348 else
3349 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3350
3351 u32GuestCR0 |= uSetCR0;
3352 u32GuestCR0 &= uZapCR0;
3353 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3354
3355 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3356 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3357 AssertRCReturn(rc, rc);
3358 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3359 AssertRCReturn(rc, rc);
3360 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3361
3362 /*
3363 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3364 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3365 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3366 */
3367 uint32_t u32CR0Mask = 0;
3368 u32CR0Mask = X86_CR0_PE
3369 | X86_CR0_NE
3370 | X86_CR0_WP
3371 | X86_CR0_PG
3372 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3373 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3374 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3375
3376 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3377 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3378 * and @bugref{6944}. */
3379#if 0
3380 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3381 u32CR0Mask &= ~X86_CR0_PE;
3382#endif
3383 if (pVM->hm.s.fNestedPaging)
3384 u32CR0Mask &= ~X86_CR0_WP;
3385
3386 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3387 if (fInterceptNM)
3388 {
3389 u32CR0Mask |= X86_CR0_TS
3390 | X86_CR0_MP;
3391 }
3392
3393 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3394 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3395 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3396 AssertRCReturn(rc, rc);
3397 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3398
3399 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3400 }
3401 return rc;
3402}
3403
3404
3405/**
3406 * Loads the guest control registers (CR3, CR4) into the guest-state area
3407 * in the VMCS.
3408 *
3409 * @returns VBox status code.
3410 * @param pVM Pointer to the VM.
3411 * @param pVCpu Pointer to the VMCPU.
3412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3413 * out-of-sync. Make sure to update the required fields
3414 * before using them.
3415 *
3416 * @remarks No-long-jump zone!!!
3417 */
3418static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3419{
3420 int rc = VINF_SUCCESS;
3421 PVM pVM = pVCpu->CTX_SUFF(pVM);
3422
3423 /*
3424 * Guest CR2.
3425 * It's always loaded in the assembler code. Nothing to do here.
3426 */
3427
3428 /*
3429 * Guest CR3.
3430 */
3431 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3432 {
3433 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3434 if (pVM->hm.s.fNestedPaging)
3435 {
3436 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3437
3438 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3439 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3440 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3441 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3442
3443 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3444 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3445 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3446
3447 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3448 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3449 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3450 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3451
3452 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3453 AssertRCReturn(rc, rc);
3454 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3455
3456 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3457 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3458 {
3459 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3460 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3461 {
3462 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3463 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3464 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3465 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3466 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3467 }
3468
3469 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3470 have Unrestricted Execution to handle the guest when it's not using paging. */
3471 GCPhysGuestCR3 = pMixedCtx->cr3;
3472 }
3473 else
3474 {
3475 /*
3476 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3477 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3478 * EPT takes care of translating it to host-physical addresses.
3479 */
3480 RTGCPHYS GCPhys;
3481 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3482 Assert(PDMVmmDevHeapIsEnabled(pVM));
3483
3484 /* We obtain it here every time as the guest could have relocated this PCI region. */
3485 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3486 AssertRCReturn(rc, rc);
3487
3488 GCPhysGuestCR3 = GCPhys;
3489 }
3490
3491 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3492 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3493 }
3494 else
3495 {
3496 /* Non-nested paging case, just use the hypervisor's CR3. */
3497 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3498
3499 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3500 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3501 }
3502 AssertRCReturn(rc, rc);
3503
3504 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3505 }
3506
3507 /*
3508 * Guest CR4.
3509 */
3510 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3511 {
3512 Assert(!(pMixedCtx->cr4 >> 32));
3513 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3514
3515 /* The guest's view of its CR4 is unblemished. */
3516 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3517 AssertRCReturn(rc, rc);
3518 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3519
3520 /* Setup VT-x's view of the guest CR4. */
3521 /*
3522 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3523 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3524 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3525 */
3526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3527 {
3528 Assert(pVM->hm.s.vmx.pRealModeTSS);
3529 Assert(PDMVmmDevHeapIsEnabled(pVM));
3530 u32GuestCR4 &= ~X86_CR4_VME;
3531 }
3532
3533 if (pVM->hm.s.fNestedPaging)
3534 {
3535 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3536 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3537 {
3538 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3539 u32GuestCR4 |= X86_CR4_PSE;
3540 /* Our identity mapping is a 32 bits page directory. */
3541 u32GuestCR4 &= ~X86_CR4_PAE;
3542 }
3543 /* else use guest CR4.*/
3544 }
3545 else
3546 {
3547 /*
3548 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3549 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3550 */
3551 switch (pVCpu->hm.s.enmShadowMode)
3552 {
3553 case PGMMODE_REAL: /* Real-mode. */
3554 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3555 case PGMMODE_32_BIT: /* 32-bit paging. */
3556 {
3557 u32GuestCR4 &= ~X86_CR4_PAE;
3558 break;
3559 }
3560
3561 case PGMMODE_PAE: /* PAE paging. */
3562 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3563 {
3564 u32GuestCR4 |= X86_CR4_PAE;
3565 break;
3566 }
3567
3568 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3569 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3570#ifdef VBOX_ENABLE_64_BITS_GUESTS
3571 break;
3572#endif
3573 default:
3574 AssertFailed();
3575 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3576 }
3577 }
3578
3579 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3580 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3581 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3582 u32GuestCR4 |= uSetCR4;
3583 u32GuestCR4 &= uZapCR4;
3584
3585 /* Write VT-x's view of the guest CR4 into the VMCS. */
3586 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3587 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3588 AssertRCReturn(rc, rc);
3589
3590 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3591 uint32_t u32CR4Mask = 0;
3592 u32CR4Mask = X86_CR4_VME
3593 | X86_CR4_PAE
3594 | X86_CR4_PGE
3595 | X86_CR4_PSE
3596 | X86_CR4_VMXE;
3597 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3598 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3599 AssertRCReturn(rc, rc);
3600
3601 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3602 }
3603 return rc;
3604}
3605
3606
3607/**
3608 * Loads the guest debug registers into the guest-state area in the VMCS.
3609 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3610 *
3611 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu Pointer to the VMCPU.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3624 return VINF_SUCCESS;
3625
3626#ifdef VBOX_STRICT
3627 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3628 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3629 {
3630 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3631 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3632 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3633 }
3634#endif
3635
3636 int rc;
3637 PVM pVM = pVCpu->CTX_SUFF(pVM);
3638 bool fInterceptDB = false;
3639 bool fInterceptMovDRx = false;
3640 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3641 {
3642 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3643 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3644 {
3645 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3646 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3647 AssertRCReturn(rc, rc);
3648 Assert(fInterceptDB == false);
3649 }
3650 else
3651 {
3652 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3653 pVCpu->hm.s.fClearTrapFlag = true;
3654 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3655 fInterceptDB = true;
3656 }
3657 }
3658
3659 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3660 {
3661 /*
3662 * Use the combined guest and host DRx values found in the hypervisor
3663 * register set because the debugger has breakpoints active or someone
3664 * is single stepping on the host side without a monitor trap flag.
3665 *
3666 * Note! DBGF expects a clean DR6 state before executing guest code.
3667 */
3668#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3669 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3670 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3671 {
3672 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3673 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3674 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3675 }
3676 else
3677#endif
3678 if (!CPUMIsHyperDebugStateActive(pVCpu))
3679 {
3680 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3681 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3682 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3683 }
3684
3685 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3686 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3687 AssertRCReturn(rc, rc);
3688
3689 pVCpu->hm.s.fUsingHyperDR7 = true;
3690 fInterceptDB = true;
3691 fInterceptMovDRx = true;
3692 }
3693 else
3694 {
3695 /*
3696 * If the guest has enabled debug registers, we need to load them prior to
3697 * executing guest code so they'll trigger at the right time.
3698 */
3699 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3700 {
3701#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3702 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3703 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3704 {
3705 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3706 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3707 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3708 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3709 }
3710 else
3711#endif
3712 if (CPUMIsGuestDebugStateActive(pVCpu))
3713 {
3714 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3715 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3716 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3717 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3718 }
3719 }
3720 /*
3721 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3722 * must intercept #DB in order to maintain a correct DR6 guest value.
3723 */
3724#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3725 else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3726 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3727 || !CPUMIsGuestDebugStateActive(pVCpu))
3728#else
3729 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3730#endif
3731 {
3732 fInterceptMovDRx = true;
3733 fInterceptDB = true;
3734 }
3735
3736 /* Update guest DR7. */
3737 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3738 AssertRCReturn(rc, rc);
3739
3740 pVCpu->hm.s.fUsingHyperDR7 = false;
3741 }
3742
3743 /*
3744 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3745 */
3746 if (fInterceptDB)
3747 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3748 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3749 {
3750#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3751 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3752#endif
3753 }
3754 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3755 AssertRCReturn(rc, rc);
3756
3757 /*
3758 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3759 */
3760 if (fInterceptMovDRx)
3761 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3762 else
3763 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3764 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3765 AssertRCReturn(rc, rc);
3766
3767 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3768 return VINF_SUCCESS;
3769}
3770
3771
3772#ifdef VBOX_STRICT
3773/**
3774 * Strict function to validate segment registers.
3775 *
3776 * @remarks ASSUMES CR0 is up to date.
3777 */
3778static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3779{
3780 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3781 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3782 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3783 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3784 && ( !CPUMIsGuestInRealModeEx(pCtx)
3785 && !CPUMIsGuestInV86ModeEx(pCtx)))
3786 {
3787 /* Protected mode checks */
3788 /* CS */
3789 Assert(pCtx->cs.Attr.n.u1Present);
3790 Assert(!(pCtx->cs.Attr.u & 0xf00));
3791 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3792 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3793 || !(pCtx->cs.Attr.n.u1Granularity));
3794 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3795 || (pCtx->cs.Attr.n.u1Granularity));
3796 /* CS cannot be loaded with NULL in protected mode. */
3797 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3798 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3799 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3800 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3801 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3802 else
3803 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3804 /* SS */
3805 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3806 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3807 if ( !(pCtx->cr0 & X86_CR0_PE)
3808 || pCtx->cs.Attr.n.u4Type == 3)
3809 {
3810 Assert(!pCtx->ss.Attr.n.u2Dpl);
3811 }
3812 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3813 {
3814 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3815 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3816 Assert(pCtx->ss.Attr.n.u1Present);
3817 Assert(!(pCtx->ss.Attr.u & 0xf00));
3818 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3819 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3820 || !(pCtx->ss.Attr.n.u1Granularity));
3821 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3822 || (pCtx->ss.Attr.n.u1Granularity));
3823 }
3824 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3825 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3826 {
3827 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3828 Assert(pCtx->ds.Attr.n.u1Present);
3829 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3830 Assert(!(pCtx->ds.Attr.u & 0xf00));
3831 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3832 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3833 || !(pCtx->ds.Attr.n.u1Granularity));
3834 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3835 || (pCtx->ds.Attr.n.u1Granularity));
3836 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3837 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3838 }
3839 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3840 {
3841 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3842 Assert(pCtx->es.Attr.n.u1Present);
3843 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3844 Assert(!(pCtx->es.Attr.u & 0xf00));
3845 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3846 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3847 || !(pCtx->es.Attr.n.u1Granularity));
3848 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3849 || (pCtx->es.Attr.n.u1Granularity));
3850 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3851 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3852 }
3853 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3854 {
3855 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3856 Assert(pCtx->fs.Attr.n.u1Present);
3857 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3858 Assert(!(pCtx->fs.Attr.u & 0xf00));
3859 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3860 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3861 || !(pCtx->fs.Attr.n.u1Granularity));
3862 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3863 || (pCtx->fs.Attr.n.u1Granularity));
3864 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3865 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3866 }
3867 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3868 {
3869 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3870 Assert(pCtx->gs.Attr.n.u1Present);
3871 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3872 Assert(!(pCtx->gs.Attr.u & 0xf00));
3873 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3874 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3875 || !(pCtx->gs.Attr.n.u1Granularity));
3876 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3877 || (pCtx->gs.Attr.n.u1Granularity));
3878 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3879 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3880 }
3881 /* 64-bit capable CPUs. */
3882# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3883 Assert(!(pCtx->cs.u64Base >> 32));
3884 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3885 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3886 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3887# endif
3888 }
3889 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3890 || ( CPUMIsGuestInRealModeEx(pCtx)
3891 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3892 {
3893 /* Real and v86 mode checks. */
3894 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3895 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3896 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3897 {
3898 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3899 }
3900 else
3901 {
3902 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3903 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3904 }
3905
3906 /* CS */
3907 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3908 Assert(pCtx->cs.u32Limit == 0xffff);
3909 Assert(u32CSAttr == 0xf3);
3910 /* SS */
3911 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3912 Assert(pCtx->ss.u32Limit == 0xffff);
3913 Assert(u32SSAttr == 0xf3);
3914 /* DS */
3915 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3916 Assert(pCtx->ds.u32Limit == 0xffff);
3917 Assert(u32DSAttr == 0xf3);
3918 /* ES */
3919 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3920 Assert(pCtx->es.u32Limit == 0xffff);
3921 Assert(u32ESAttr == 0xf3);
3922 /* FS */
3923 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3924 Assert(pCtx->fs.u32Limit == 0xffff);
3925 Assert(u32FSAttr == 0xf3);
3926 /* GS */
3927 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3928 Assert(pCtx->gs.u32Limit == 0xffff);
3929 Assert(u32GSAttr == 0xf3);
3930 /* 64-bit capable CPUs. */
3931# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3932 Assert(!(pCtx->cs.u64Base >> 32));
3933 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3934 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3935 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3936# endif
3937 }
3938}
3939#endif /* VBOX_STRICT */
3940
3941
3942/**
3943 * Writes a guest segment register into the guest-state area in the VMCS.
3944 *
3945 * @returns VBox status code.
3946 * @param pVCpu Pointer to the VMCPU.
3947 * @param idxSel Index of the selector in the VMCS.
3948 * @param idxLimit Index of the segment limit in the VMCS.
3949 * @param idxBase Index of the segment base in the VMCS.
3950 * @param idxAccess Index of the access rights of the segment in the VMCS.
3951 * @param pSelReg Pointer to the segment selector.
3952 * @param pCtx Pointer to the guest-CPU context.
3953 *
3954 * @remarks No-long-jump zone!!!
3955 */
3956static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3957 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3958{
3959 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3960 AssertRCReturn(rc, rc);
3961 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3962 AssertRCReturn(rc, rc);
3963 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3964 AssertRCReturn(rc, rc);
3965
3966 uint32_t u32Access = pSelReg->Attr.u;
3967 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3968 {
3969 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3970 u32Access = 0xf3;
3971 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3972 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3973 }
3974 else
3975 {
3976 /*
3977 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3978 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3979 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3980 * loaded in protected-mode have their attribute as 0.
3981 */
3982 if (!u32Access)
3983 u32Access = X86DESCATTR_UNUSABLE;
3984 }
3985
3986 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3987 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3988 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3989
3990 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3991 AssertRCReturn(rc, rc);
3992 return rc;
3993}
3994
3995
3996/**
3997 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3998 * into the guest-state area in the VMCS.
3999 *
4000 * @returns VBox status code.
4001 * @param pVM Pointer to the VM.
4002 * @param pVCPU Pointer to the VMCPU.
4003 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4004 * out-of-sync. Make sure to update the required fields
4005 * before using them.
4006 *
4007 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4008 * @remarks No-long-jump zone!!!
4009 */
4010static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4011{
4012 int rc = VERR_INTERNAL_ERROR_5;
4013 PVM pVM = pVCpu->CTX_SUFF(pVM);
4014
4015 /*
4016 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4017 */
4018 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4019 {
4020 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4021 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4022 {
4023 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4024 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4025 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4026 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4027 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4028 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4029 }
4030
4031#ifdef VBOX_WITH_REM
4032 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4033 {
4034 Assert(pVM->hm.s.vmx.pRealModeTSS);
4035 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4036 if ( pVCpu->hm.s.vmx.fWasInRealMode
4037 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4038 {
4039 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4040 in real-mode (e.g. OpenBSD 4.0) */
4041 REMFlushTBs(pVM);
4042 Log4(("Load: Switch to protected mode detected!\n"));
4043 pVCpu->hm.s.vmx.fWasInRealMode = false;
4044 }
4045 }
4046#endif
4047 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4048 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
4049 AssertRCReturn(rc, rc);
4050 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4051 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
4052 AssertRCReturn(rc, rc);
4053 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4054 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
4055 AssertRCReturn(rc, rc);
4056 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4057 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
4058 AssertRCReturn(rc, rc);
4059 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4060 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
4061 AssertRCReturn(rc, rc);
4062 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4063 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
4064 AssertRCReturn(rc, rc);
4065
4066#ifdef VBOX_STRICT
4067 /* Validate. */
4068 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4069#endif
4070
4071 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4072 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4073 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4074 }
4075
4076 /*
4077 * Guest TR.
4078 */
4079 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4080 {
4081 /*
4082 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4083 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4084 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4085 */
4086 uint16_t u16Sel = 0;
4087 uint32_t u32Limit = 0;
4088 uint64_t u64Base = 0;
4089 uint32_t u32AccessRights = 0;
4090
4091 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4092 {
4093 u16Sel = pMixedCtx->tr.Sel;
4094 u32Limit = pMixedCtx->tr.u32Limit;
4095 u64Base = pMixedCtx->tr.u64Base;
4096 u32AccessRights = pMixedCtx->tr.Attr.u;
4097 }
4098 else
4099 {
4100 Assert(pVM->hm.s.vmx.pRealModeTSS);
4101 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4102
4103 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4104 RTGCPHYS GCPhys;
4105 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4106 AssertRCReturn(rc, rc);
4107
4108 X86DESCATTR DescAttr;
4109 DescAttr.u = 0;
4110 DescAttr.n.u1Present = 1;
4111 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4112
4113 u16Sel = 0;
4114 u32Limit = HM_VTX_TSS_SIZE;
4115 u64Base = GCPhys; /* in real-mode phys = virt. */
4116 u32AccessRights = DescAttr.u;
4117 }
4118
4119 /* Validate. */
4120 Assert(!(u16Sel & RT_BIT(2)));
4121 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4122 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4123 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4124 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4125 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4126 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4127 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4128 Assert( (u32Limit & 0xfff) == 0xfff
4129 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4130 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4131 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4132
4133 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4134 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4135 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4136 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4137
4138 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4139 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4140 }
4141
4142 /*
4143 * Guest GDTR.
4144 */
4145 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4146 {
4147 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4148 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4149
4150 /* Validate. */
4151 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4152
4153 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4154 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4155 }
4156
4157 /*
4158 * Guest LDTR.
4159 */
4160 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4161 {
4162 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4163 uint32_t u32Access = 0;
4164 if (!pMixedCtx->ldtr.Attr.u)
4165 u32Access = X86DESCATTR_UNUSABLE;
4166 else
4167 u32Access = pMixedCtx->ldtr.Attr.u;
4168
4169 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4170 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4171 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4172 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4173
4174 /* Validate. */
4175 if (!(u32Access & X86DESCATTR_UNUSABLE))
4176 {
4177 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4178 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4179 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4180 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4181 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4182 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4183 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4184 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4185 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4186 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4187 }
4188
4189 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4190 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4191 }
4192
4193 /*
4194 * Guest IDTR.
4195 */
4196 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4197 {
4198 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4199 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4200
4201 /* Validate. */
4202 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4203
4204 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4205 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4206 }
4207
4208 return VINF_SUCCESS;
4209}
4210
4211
4212/**
4213 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4214 * areas. These MSRs will automatically be loaded to the host CPU on every
4215 * successful VM entry and stored from the host CPU on every successful VM exit.
4216 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4217 *
4218 * @returns VBox status code.
4219 * @param pVCpu Pointer to the VMCPU.
4220 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4221 * out-of-sync. Make sure to update the required fields
4222 * before using them.
4223 *
4224 * @remarks No-long-jump zone!!!
4225 */
4226static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4227{
4228 AssertPtr(pVCpu);
4229 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4230
4231 /*
4232 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4233 */
4234 int rc = VINF_SUCCESS;
4235 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4236 {
4237#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4238 PVM pVM = pVCpu->CTX_SUFF(pVM);
4239 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4240 uint32_t cGuestMsrs = 0;
4241
4242 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4243 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4244 * when the guest really is in 64-bit mode. */
4245 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4246 if (fSupportsLongMode)
4247 {
4248 pGuestMsr->u32Msr = MSR_K8_LSTAR;
4249 pGuestMsr->u32Reserved = 0;
4250 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4251 pGuestMsr++; cGuestMsrs++;
4252 pGuestMsr->u32Msr = MSR_K6_STAR;
4253 pGuestMsr->u32Reserved = 0;
4254 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4255 pGuestMsr++; cGuestMsrs++;
4256 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4257 pGuestMsr->u32Reserved = 0;
4258 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4259 pGuestMsr++; cGuestMsrs++;
4260 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4261 pGuestMsr->u32Reserved = 0;
4262 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4263 pGuestMsr++; cGuestMsrs++;
4264 }
4265
4266 /*
4267 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4268 * load the guest's copy always (since the MSR bitmap allows passthru unconditionally).
4269 */
4270 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
4271 {
4272 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4273 pGuestMsr->u32Reserved = 0;
4274 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4275 AssertRCReturn(rc, rc);
4276 pGuestMsr++; cGuestMsrs++;
4277 }
4278
4279 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4280 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4281 {
4282 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4283 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4284 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4285 }
4286
4287 /* Update the VCPU's copy of the guest MSR count. */
4288 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4289 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4290 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4291#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4292
4293 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4294 }
4295
4296 /*
4297 * Guest Sysenter MSRs.
4298 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4299 * VM-exits on WRMSRs for these MSRs.
4300 */
4301 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4302 {
4303 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4304 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4305 }
4306
4307 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4308 {
4309 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4310 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4311 }
4312
4313 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4314 {
4315 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4316 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4317 }
4318
4319 return rc;
4320}
4321
4322
4323/**
4324 * Loads the guest activity state into the guest-state area in the VMCS.
4325 *
4326 * @returns VBox status code.
4327 * @param pVCpu Pointer to the VMCPU.
4328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4329 * out-of-sync. Make sure to update the required fields
4330 * before using them.
4331 *
4332 * @remarks No-long-jump zone!!!
4333 */
4334static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4335{
4336 /** @todo See if we can make use of other states, e.g.
4337 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4338 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4339 {
4340 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4341 AssertRCReturn(rc, rc);
4342
4343 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4344 }
4345 return VINF_SUCCESS;
4346}
4347
4348
4349/**
4350 * Sets up the appropriate function to run guest code.
4351 *
4352 * @returns VBox status code.
4353 * @param pVCpu Pointer to the VMCPU.
4354 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4355 * out-of-sync. Make sure to update the required fields
4356 * before using them.
4357 *
4358 * @remarks No-long-jump zone!!!
4359 */
4360static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4361{
4362 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4363 {
4364#ifndef VBOX_ENABLE_64_BITS_GUESTS
4365 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4366#endif
4367 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4368#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4369 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4370 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4371 {
4372 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4373 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4374 }
4375#else
4376 /* 64-bit host or hybrid host. */
4377 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4378#endif
4379 }
4380 else
4381 {
4382 /* Guest is not in long mode, use the 32-bit handler. */
4383#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4384 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4385 {
4386 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4387 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4388 }
4389#else
4390 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4391#endif
4392 }
4393 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4394 return VINF_SUCCESS;
4395}
4396
4397
4398/**
4399 * Wrapper for running the guest code in VT-x.
4400 *
4401 * @returns VBox strict status code.
4402 * @param pVM Pointer to the VM.
4403 * @param pVCpu Pointer to the VMCPU.
4404 * @param pCtx Pointer to the guest-CPU context.
4405 *
4406 * @remarks No-long-jump zone!!!
4407 */
4408DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4409{
4410 /*
4411 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4412 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4413 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4414 */
4415 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4416 /** @todo Add stats for resume vs launch. */
4417#ifdef VBOX_WITH_KERNEL_USING_XMM
4418 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4419#else
4420 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4421#endif
4422}
4423
4424
4425/**
4426 * Reports world-switch error and dumps some useful debug info.
4427 *
4428 * @param pVM Pointer to the VM.
4429 * @param pVCpu Pointer to the VMCPU.
4430 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4431 * @param pCtx Pointer to the guest-CPU context.
4432 * @param pVmxTransient Pointer to the VMX transient structure (only
4433 * exitReason updated).
4434 */
4435static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4436{
4437 Assert(pVM);
4438 Assert(pVCpu);
4439 Assert(pCtx);
4440 Assert(pVmxTransient);
4441 HMVMX_ASSERT_PREEMPT_SAFE();
4442
4443 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4444 switch (rcVMRun)
4445 {
4446 case VERR_VMX_INVALID_VMXON_PTR:
4447 AssertFailed();
4448 break;
4449 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4450 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4451 {
4452 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4453 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4454 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4455 AssertRC(rc);
4456
4457 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4458 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4459 Cannot do it here as we may have been long preempted. */
4460
4461#ifdef VBOX_STRICT
4462 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4463 pVmxTransient->uExitReason));
4464 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4465 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4466 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4467 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4468 else
4469 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4470 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4471 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4472
4473 /* VMX control bits. */
4474 uint32_t u32Val;
4475 uint64_t u64Val;
4476 HMVMXHCUINTREG uHCReg;
4477 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4478 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4479 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4480 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4481 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4482 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4483 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4484 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4485 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4486 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4487 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4488 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4489 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4490 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4491 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4492 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4493 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4494 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4495 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4496 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4497 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4498 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4499 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4500 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4501 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4502 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4503 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4504 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4505 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4506 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4507 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4508 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4509 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4510 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4511 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4512 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4513 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4514 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4515 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4516 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4517 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4518 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4519
4520 /* Guest bits. */
4521 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4522 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4523 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4524 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4525 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4526 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4527 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4528 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4529
4530 /* Host bits. */
4531 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4532 Log4(("Host CR0 %#RHr\n", uHCReg));
4533 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4534 Log4(("Host CR3 %#RHr\n", uHCReg));
4535 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4536 Log4(("Host CR4 %#RHr\n", uHCReg));
4537
4538 RTGDTR HostGdtr;
4539 PCX86DESCHC pDesc;
4540 ASMGetGDTR(&HostGdtr);
4541 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4542 Log4(("Host CS %#08x\n", u32Val));
4543 if (u32Val < HostGdtr.cbGdt)
4544 {
4545 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4546 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4547 }
4548
4549 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4550 Log4(("Host DS %#08x\n", u32Val));
4551 if (u32Val < HostGdtr.cbGdt)
4552 {
4553 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4554 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4555 }
4556
4557 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4558 Log4(("Host ES %#08x\n", u32Val));
4559 if (u32Val < HostGdtr.cbGdt)
4560 {
4561 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4562 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4563 }
4564
4565 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4566 Log4(("Host FS %#08x\n", u32Val));
4567 if (u32Val < HostGdtr.cbGdt)
4568 {
4569 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4570 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4571 }
4572
4573 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4574 Log4(("Host GS %#08x\n", u32Val));
4575 if (u32Val < HostGdtr.cbGdt)
4576 {
4577 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4578 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4579 }
4580
4581 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4582 Log4(("Host SS %#08x\n", u32Val));
4583 if (u32Val < HostGdtr.cbGdt)
4584 {
4585 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4586 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4587 }
4588
4589 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4590 Log4(("Host TR %#08x\n", u32Val));
4591 if (u32Val < HostGdtr.cbGdt)
4592 {
4593 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4594 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4595 }
4596
4597 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4598 Log4(("Host TR Base %#RHv\n", uHCReg));
4599 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4600 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4601 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4602 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4603 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4604 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4605 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4606 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4607 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4608 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4609 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4610 Log4(("Host RSP %#RHv\n", uHCReg));
4611 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4612 Log4(("Host RIP %#RHv\n", uHCReg));
4613# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4614 if (HMVMX_IS_64BIT_HOST_MODE())
4615 {
4616 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4617 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4618 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4619 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4620 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4621 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4622 }
4623# endif
4624#endif /* VBOX_STRICT */
4625 break;
4626 }
4627
4628 default:
4629 /* Impossible */
4630 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4631 break;
4632 }
4633 NOREF(pVM);
4634}
4635
4636
4637#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4638#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4639# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4640#endif
4641#ifdef VBOX_STRICT
4642static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4643{
4644 switch (idxField)
4645 {
4646 case VMX_VMCS_GUEST_RIP:
4647 case VMX_VMCS_GUEST_RSP:
4648 case VMX_VMCS_GUEST_SYSENTER_EIP:
4649 case VMX_VMCS_GUEST_SYSENTER_ESP:
4650 case VMX_VMCS_GUEST_GDTR_BASE:
4651 case VMX_VMCS_GUEST_IDTR_BASE:
4652 case VMX_VMCS_GUEST_CS_BASE:
4653 case VMX_VMCS_GUEST_DS_BASE:
4654 case VMX_VMCS_GUEST_ES_BASE:
4655 case VMX_VMCS_GUEST_FS_BASE:
4656 case VMX_VMCS_GUEST_GS_BASE:
4657 case VMX_VMCS_GUEST_SS_BASE:
4658 case VMX_VMCS_GUEST_LDTR_BASE:
4659 case VMX_VMCS_GUEST_TR_BASE:
4660 case VMX_VMCS_GUEST_CR3:
4661 return true;
4662 }
4663 return false;
4664}
4665
4666static bool hmR0VmxIsValidReadField(uint32_t idxField)
4667{
4668 switch (idxField)
4669 {
4670 /* Read-only fields. */
4671 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4672 return true;
4673 }
4674 /* Remaining readable fields should also be writable. */
4675 return hmR0VmxIsValidWriteField(idxField);
4676}
4677#endif /* VBOX_STRICT */
4678
4679
4680/**
4681 * Executes the specified handler in 64-bit mode.
4682 *
4683 * @returns VBox status code.
4684 * @param pVM Pointer to the VM.
4685 * @param pVCpu Pointer to the VMCPU.
4686 * @param pCtx Pointer to the guest CPU context.
4687 * @param enmOp The operation to perform.
4688 * @param cbParam Number of parameters.
4689 * @param paParam Array of 32-bit parameters.
4690 */
4691VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4692 uint32_t *paParam)
4693{
4694 int rc, rc2;
4695 PHMGLOBALCPUINFO pCpu;
4696 RTHCPHYS HCPhysCpuPage;
4697 RTCCUINTREG uOldEflags;
4698
4699 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4700 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4701 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4702 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4703
4704#ifdef VBOX_STRICT
4705 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4706 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4707
4708 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4709 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4710#endif
4711
4712 /* Disable interrupts. */
4713 uOldEflags = ASMIntDisableFlags();
4714
4715#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4716 RTCPUID idHostCpu = RTMpCpuId();
4717 CPUMR0SetLApic(pVCpu, idHostCpu);
4718#endif
4719
4720 pCpu = HMR0GetCurrentCpu();
4721 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4722
4723 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4724 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4725
4726 /* Leave VMX Root Mode. */
4727 VMXDisable();
4728
4729 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4730
4731 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4732 CPUMSetHyperEIP(pVCpu, enmOp);
4733 for (int i = (int)cbParam - 1; i >= 0; i--)
4734 CPUMPushHyper(pVCpu, paParam[i]);
4735
4736 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4737
4738 /* Call the switcher. */
4739 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4740 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4741
4742 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4743 /* Make sure the VMX instructions don't cause #UD faults. */
4744 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4745
4746 /* Re-enter VMX Root Mode */
4747 rc2 = VMXEnable(HCPhysCpuPage);
4748 if (RT_FAILURE(rc2))
4749 {
4750 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4751 ASMSetFlags(uOldEflags);
4752 return rc2;
4753 }
4754
4755 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4756 AssertRC(rc2);
4757 Assert(!(ASMGetFlags() & X86_EFL_IF));
4758 ASMSetFlags(uOldEflags);
4759 return rc;
4760}
4761
4762
4763/**
4764 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4765 * supporting 64-bit guests.
4766 *
4767 * @returns VBox status code.
4768 * @param fResume Whether to VMLAUNCH or VMRESUME.
4769 * @param pCtx Pointer to the guest-CPU context.
4770 * @param pCache Pointer to the VMCS cache.
4771 * @param pVM Pointer to the VM.
4772 * @param pVCpu Pointer to the VMCPU.
4773 */
4774DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4775{
4776 uint32_t aParam[6];
4777 PHMGLOBALCPUINFO pCpu = NULL;
4778 RTHCPHYS HCPhysCpuPage = 0;
4779 int rc = VERR_INTERNAL_ERROR_5;
4780
4781 pCpu = HMR0GetCurrentCpu();
4782 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4783
4784#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4785 pCache->uPos = 1;
4786 pCache->interPD = PGMGetInterPaeCR3(pVM);
4787 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4788#endif
4789
4790#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4791 pCache->TestIn.HCPhysCpuPage = 0;
4792 pCache->TestIn.HCPhysVmcs = 0;
4793 pCache->TestIn.pCache = 0;
4794 pCache->TestOut.HCPhysVmcs = 0;
4795 pCache->TestOut.pCache = 0;
4796 pCache->TestOut.pCtx = 0;
4797 pCache->TestOut.eflags = 0;
4798#endif
4799
4800 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4801 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4802 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4803 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4804 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4805 aParam[5] = 0;
4806
4807#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4808 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4809 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4810#endif
4811 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4812
4813#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4814 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4815 Assert(pCtx->dr[4] == 10);
4816 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4817#endif
4818
4819#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4820 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4821 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4822 pVCpu->hm.s.vmx.HCPhysVmcs));
4823 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4824 pCache->TestOut.HCPhysVmcs));
4825 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4826 pCache->TestOut.pCache));
4827 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4828 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4829 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4830 pCache->TestOut.pCtx));
4831 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4832#endif
4833 return rc;
4834}
4835
4836
4837/**
4838 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4839 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4840 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4841 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4842 *
4843 * @returns VBox status code.
4844 * @param pVM Pointer to the VM.
4845 * @param pVCpu Pointer to the VMCPU.
4846 */
4847static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4848{
4849#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4850{ \
4851 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4852 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4853 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4854 ++cReadFields; \
4855}
4856
4857 AssertPtr(pVM);
4858 AssertPtr(pVCpu);
4859 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4860 uint32_t cReadFields = 0;
4861
4862 /*
4863 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4864 * and serve to indicate exceptions to the rules.
4865 */
4866
4867 /* Guest-natural selector base fields. */
4868#if 0
4869 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4870 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4871 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4872#endif
4873 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4874 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4875 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4876 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4877 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4878 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4879 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4880 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4881 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4882 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4883 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4884 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4885#if 0
4886 /* Unused natural width guest-state fields. */
4887 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4888 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4889#endif
4890 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4891 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4892
4893 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4894#if 0
4895 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4896 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4897 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4898 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4899 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4900 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4901 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4902 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4903 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4904#endif
4905
4906 /* Natural width guest-state fields. */
4907 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4908#if 0
4909 /* Currently unused field. */
4910 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4911#endif
4912
4913 if (pVM->hm.s.fNestedPaging)
4914 {
4915 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4916 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4917 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4918 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4919 }
4920 else
4921 {
4922 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4923 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4924 }
4925
4926#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4927 return VINF_SUCCESS;
4928}
4929
4930
4931/**
4932 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4933 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4934 * darwin, running 64-bit guests).
4935 *
4936 * @returns VBox status code.
4937 * @param pVCpu Pointer to the VMCPU.
4938 * @param idxField The VMCS field encoding.
4939 * @param u64Val 16, 32 or 64 bits value.
4940 */
4941VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4942{
4943 int rc;
4944 switch (idxField)
4945 {
4946 /*
4947 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4948 */
4949 /* 64-bit Control fields. */
4950 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4951 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4952 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4953 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4954 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4955 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4956 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4957 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4958 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4959 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4960 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4961 case VMX_VMCS64_CTRL_EPTP_FULL:
4962 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4963 /* 64-bit Guest-state fields. */
4964 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4965 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4966 case VMX_VMCS64_GUEST_PAT_FULL:
4967 case VMX_VMCS64_GUEST_EFER_FULL:
4968 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4969 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4970 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4971 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4972 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4973 /* 64-bit Host-state fields. */
4974 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4975 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4976 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4977 {
4978 rc = VMXWriteVmcs32(idxField, u64Val);
4979 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4980 break;
4981 }
4982
4983 /*
4984 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4985 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4986 */
4987 /* Natural-width Guest-state fields. */
4988 case VMX_VMCS_GUEST_CR3:
4989 case VMX_VMCS_GUEST_ES_BASE:
4990 case VMX_VMCS_GUEST_CS_BASE:
4991 case VMX_VMCS_GUEST_SS_BASE:
4992 case VMX_VMCS_GUEST_DS_BASE:
4993 case VMX_VMCS_GUEST_FS_BASE:
4994 case VMX_VMCS_GUEST_GS_BASE:
4995 case VMX_VMCS_GUEST_LDTR_BASE:
4996 case VMX_VMCS_GUEST_TR_BASE:
4997 case VMX_VMCS_GUEST_GDTR_BASE:
4998 case VMX_VMCS_GUEST_IDTR_BASE:
4999 case VMX_VMCS_GUEST_RSP:
5000 case VMX_VMCS_GUEST_RIP:
5001 case VMX_VMCS_GUEST_SYSENTER_ESP:
5002 case VMX_VMCS_GUEST_SYSENTER_EIP:
5003 {
5004 if (!(u64Val >> 32))
5005 {
5006 /* If this field is 64-bit, VT-x will zero out the top bits. */
5007 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5008 }
5009 else
5010 {
5011 /* Assert that only the 32->64 switcher case should ever come here. */
5012 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5013 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5014 }
5015 break;
5016 }
5017
5018 default:
5019 {
5020 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5021 rc = VERR_INVALID_PARAMETER;
5022 break;
5023 }
5024 }
5025 AssertRCReturn(rc, rc);
5026 return rc;
5027}
5028
5029
5030/**
5031 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5032 * hosts (except darwin) for 64-bit guests.
5033 *
5034 * @param pVCpu Pointer to the VMCPU.
5035 * @param idxField The VMCS field encoding.
5036 * @param u64Val 16, 32 or 64 bits value.
5037 */
5038VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5039{
5040 AssertPtr(pVCpu);
5041 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5042
5043 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5044 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5045
5046 /* Make sure there are no duplicates. */
5047 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5048 {
5049 if (pCache->Write.aField[i] == idxField)
5050 {
5051 pCache->Write.aFieldVal[i] = u64Val;
5052 return VINF_SUCCESS;
5053 }
5054 }
5055
5056 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5057 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5058 pCache->Write.cValidEntries++;
5059 return VINF_SUCCESS;
5060}
5061
5062/* Enable later when the assembly code uses these as callbacks. */
5063#if 0
5064/*
5065 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5066 *
5067 * @param pVCpu Pointer to the VMCPU.
5068 * @param pCache Pointer to the VMCS cache.
5069 *
5070 * @remarks No-long-jump zone!!!
5071 */
5072VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5073{
5074 AssertPtr(pCache);
5075 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5076 {
5077 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5078 AssertRC(rc);
5079 }
5080 pCache->Write.cValidEntries = 0;
5081}
5082
5083
5084/**
5085 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5086 *
5087 * @param pVCpu Pointer to the VMCPU.
5088 * @param pCache Pointer to the VMCS cache.
5089 *
5090 * @remarks No-long-jump zone!!!
5091 */
5092VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5093{
5094 AssertPtr(pCache);
5095 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5096 {
5097 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5098 AssertRC(rc);
5099 }
5100}
5101#endif
5102#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5103
5104
5105/**
5106 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5107 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5108 * timer.
5109 *
5110 * @returns VBox status code.
5111 * @param pVCpu Pointer to the VMCPU.
5112 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5113 * out-of-sync. Make sure to update the required fields
5114 * before using them.
5115 * @remarks No-long-jump zone!!!
5116 */
5117static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5118{
5119 int rc = VERR_INTERNAL_ERROR_5;
5120 bool fOffsettedTsc = false;
5121 PVM pVM = pVCpu->CTX_SUFF(pVM);
5122 if (pVM->hm.s.vmx.fUsePreemptTimer)
5123 {
5124 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5125
5126 /* Make sure the returned values have sane upper and lower boundaries. */
5127 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5128 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5129 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5130 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5131
5132 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5133 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5134 }
5135 else
5136 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5137
5138 if (fOffsettedTsc)
5139 {
5140 uint64_t u64CurTSC = ASMReadTSC();
5141 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5142 {
5143 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5144 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5145
5146 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5147 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5148 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5149 }
5150 else
5151 {
5152 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5153 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5154 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5155 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5156 }
5157 }
5158 else
5159 {
5160 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5161 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5162 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5163 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5164 }
5165}
5166
5167
5168/**
5169 * Determines if an exception is a contributory exception. Contributory
5170 * exceptions are ones which can cause double-faults. Page-fault is
5171 * intentionally not included here as it's a conditional contributory exception.
5172 *
5173 * @returns true if the exception is contributory, false otherwise.
5174 * @param uVector The exception vector.
5175 */
5176DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5177{
5178 switch (uVector)
5179 {
5180 case X86_XCPT_GP:
5181 case X86_XCPT_SS:
5182 case X86_XCPT_NP:
5183 case X86_XCPT_TS:
5184 case X86_XCPT_DE:
5185 return true;
5186 default:
5187 break;
5188 }
5189 return false;
5190}
5191
5192
5193/**
5194 * Sets an event as a pending event to be injected into the guest.
5195 *
5196 * @param pVCpu Pointer to the VMCPU.
5197 * @param u32IntInfo The VM-entry interruption-information field.
5198 * @param cbInstr The VM-entry instruction length in bytes (for software
5199 * interrupts, exceptions and privileged software
5200 * exceptions).
5201 * @param u32ErrCode The VM-entry exception error code.
5202 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5203 * page-fault.
5204 *
5205 * @remarks Statistics counter assumes this is a guest event being injected or
5206 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5207 * always incremented.
5208 */
5209DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5210 RTGCUINTPTR GCPtrFaultAddress)
5211{
5212 Assert(!pVCpu->hm.s.Event.fPending);
5213 pVCpu->hm.s.Event.fPending = true;
5214 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5215 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5216 pVCpu->hm.s.Event.cbInstr = cbInstr;
5217 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5218
5219 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5220}
5221
5222
5223/**
5224 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5225 *
5226 * @param pVCpu Pointer to the VMCPU.
5227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5228 * out-of-sync. Make sure to update the required fields
5229 * before using them.
5230 */
5231DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5232{
5233 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5234 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5235 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5236 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5237}
5238
5239
5240/**
5241 * Handle a condition that occurred while delivering an event through the guest
5242 * IDT.
5243 *
5244 * @returns VBox status code (informational error codes included).
5245 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5246 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5247 * continue execution of the guest which will delivery the #DF.
5248 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5249 *
5250 * @param pVCpu Pointer to the VMCPU.
5251 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5252 * out-of-sync. Make sure to update the required fields
5253 * before using them.
5254 * @param pVmxTransient Pointer to the VMX transient structure.
5255 *
5256 * @remarks No-long-jump zone!!!
5257 */
5258static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5259{
5260 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5261 AssertRCReturn(rc, rc);
5262 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5263 {
5264 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5265 AssertRCReturn(rc, rc);
5266
5267 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5268 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5269 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5270
5271 typedef enum
5272 {
5273 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5274 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5275 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5276 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5277 } VMXREFLECTXCPT;
5278
5279 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5280 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5281 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5282 {
5283 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5284 {
5285 enmReflect = VMXREFLECTXCPT_XCPT;
5286#ifdef VBOX_STRICT
5287 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5288 && uExitVector == X86_XCPT_PF)
5289 {
5290 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5291 }
5292#endif
5293 if ( uExitVector == X86_XCPT_PF
5294 && uIdtVector == X86_XCPT_PF)
5295 {
5296 pVmxTransient->fVectoringPF = true;
5297 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5298 }
5299 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5300 && hmR0VmxIsContributoryXcpt(uExitVector)
5301 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5302 || uIdtVector == X86_XCPT_PF))
5303 {
5304 enmReflect = VMXREFLECTXCPT_DF;
5305 }
5306 else if (uIdtVector == X86_XCPT_DF)
5307 enmReflect = VMXREFLECTXCPT_TF;
5308 }
5309 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5310 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5311 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5312 {
5313 /*
5314 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5315 * (whatever they are) as they reoccur when restarting the instruction.
5316 */
5317 enmReflect = VMXREFLECTXCPT_XCPT;
5318 }
5319 }
5320 else
5321 {
5322 /*
5323 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5324 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5325 * original exception to the guest after handling the VM-exit.
5326 */
5327 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5328 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5329 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5330 {
5331 enmReflect = VMXREFLECTXCPT_XCPT;
5332 }
5333 }
5334
5335 switch (enmReflect)
5336 {
5337 case VMXREFLECTXCPT_XCPT:
5338 {
5339 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5340 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5341 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5342
5343 uint32_t u32ErrCode = 0;
5344 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5345 {
5346 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5347 AssertRCReturn(rc, rc);
5348 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5349 }
5350
5351 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5352 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5353 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5354 rc = VINF_SUCCESS;
5355 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5356 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5357
5358 break;
5359 }
5360
5361 case VMXREFLECTXCPT_DF:
5362 {
5363 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5364 rc = VINF_HM_DOUBLE_FAULT;
5365 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5366 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5367
5368 break;
5369 }
5370
5371 case VMXREFLECTXCPT_TF:
5372 {
5373 rc = VINF_EM_RESET;
5374 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5375 uExitVector));
5376 break;
5377 }
5378
5379 default:
5380 Assert(rc == VINF_SUCCESS);
5381 break;
5382 }
5383 }
5384 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5385 return rc;
5386}
5387
5388
5389/**
5390 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5391 *
5392 * @returns VBox status code.
5393 * @param pVCpu Pointer to the VMCPU.
5394 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5395 * out-of-sync. Make sure to update the required fields
5396 * before using them.
5397 *
5398 * @remarks No-long-jump zone!!!
5399 */
5400static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5401{
5402 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5403 {
5404 uint32_t uVal = 0;
5405 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5406 AssertRCReturn(rc, rc);
5407
5408 uint32_t uShadow = 0;
5409 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5410 AssertRCReturn(rc, rc);
5411
5412 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5413 CPUMSetGuestCR0(pVCpu, uVal);
5414 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5415 }
5416 return VINF_SUCCESS;
5417}
5418
5419
5420/**
5421 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5422 *
5423 * @returns VBox status code.
5424 * @param pVCpu Pointer to the VMCPU.
5425 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5426 * out-of-sync. Make sure to update the required fields
5427 * before using them.
5428 *
5429 * @remarks No-long-jump zone!!!
5430 */
5431static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5432{
5433 int rc = VINF_SUCCESS;
5434 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5435 {
5436 uint32_t uVal = 0;
5437 uint32_t uShadow = 0;
5438 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5439 AssertRCReturn(rc, rc);
5440 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5441 AssertRCReturn(rc, rc);
5442
5443 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5444 CPUMSetGuestCR4(pVCpu, uVal);
5445 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5446 }
5447 return rc;
5448}
5449
5450
5451/**
5452 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5453 *
5454 * @returns VBox status code.
5455 * @param pVCpu Pointer to the VMCPU.
5456 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5457 * out-of-sync. Make sure to update the required fields
5458 * before using them.
5459 *
5460 * @remarks No-long-jump zone!!!
5461 */
5462static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5463{
5464 int rc = VINF_SUCCESS;
5465 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5466 {
5467 uint64_t u64Val = 0;
5468 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5469 AssertRCReturn(rc, rc);
5470
5471 pMixedCtx->rip = u64Val;
5472 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5473 }
5474 return rc;
5475}
5476
5477
5478/**
5479 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5480 *
5481 * @returns VBox status code.
5482 * @param pVCpu Pointer to the VMCPU.
5483 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5484 * out-of-sync. Make sure to update the required fields
5485 * before using them.
5486 *
5487 * @remarks No-long-jump zone!!!
5488 */
5489static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5490{
5491 int rc = VINF_SUCCESS;
5492 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5493 {
5494 uint64_t u64Val = 0;
5495 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5496 AssertRCReturn(rc, rc);
5497
5498 pMixedCtx->rsp = u64Val;
5499 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5500 }
5501 return rc;
5502}
5503
5504
5505/**
5506 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5507 *
5508 * @returns VBox status code.
5509 * @param pVCpu Pointer to the VMCPU.
5510 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5511 * out-of-sync. Make sure to update the required fields
5512 * before using them.
5513 *
5514 * @remarks No-long-jump zone!!!
5515 */
5516static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5517{
5518 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5519 {
5520 uint32_t uVal = 0;
5521 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5522 AssertRCReturn(rc, rc);
5523
5524 pMixedCtx->eflags.u32 = uVal;
5525 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5526 {
5527 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5528 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5529
5530 pMixedCtx->eflags.Bits.u1VM = 0;
5531 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5532 }
5533
5534 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5535 }
5536 return VINF_SUCCESS;
5537}
5538
5539
5540/**
5541 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5542 * guest-CPU context.
5543 */
5544DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5545{
5546 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5547 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5548 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5549 return rc;
5550}
5551
5552
5553/**
5554 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5555 * from the guest-state area in the VMCS.
5556 *
5557 * @param pVCpu Pointer to the VMCPU.
5558 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5559 * out-of-sync. Make sure to update the required fields
5560 * before using them.
5561 *
5562 * @remarks No-long-jump zone!!!
5563 */
5564static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5565{
5566 uint32_t uIntrState = 0;
5567 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5568 AssertRC(rc);
5569
5570 if (!uIntrState)
5571 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5572 else
5573 {
5574 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5575 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5576 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5577 AssertRC(rc);
5578 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5579 AssertRC(rc);
5580
5581 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5582 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5583 }
5584}
5585
5586
5587/**
5588 * Saves the guest's activity state.
5589 *
5590 * @returns VBox status code.
5591 * @param pVCpu Pointer to the VMCPU.
5592 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5593 * out-of-sync. Make sure to update the required fields
5594 * before using them.
5595 *
5596 * @remarks No-long-jump zone!!!
5597 */
5598static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5599{
5600 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5601 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5602 return VINF_SUCCESS;
5603}
5604
5605
5606/**
5607 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5608 * the current VMCS into the guest-CPU context.
5609 *
5610 * @returns VBox status code.
5611 * @param pVCpu Pointer to the VMCPU.
5612 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5613 * out-of-sync. Make sure to update the required fields
5614 * before using them.
5615 *
5616 * @remarks No-long-jump zone!!!
5617 */
5618static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5619{
5620 int rc = VINF_SUCCESS;
5621 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5622 {
5623 uint32_t u32Val = 0;
5624 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5625 pMixedCtx->SysEnter.cs = u32Val;
5626 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5627 }
5628
5629 uint64_t u64Val = 0;
5630 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5631 {
5632 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5633 pMixedCtx->SysEnter.eip = u64Val;
5634 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5635 }
5636 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5637 {
5638 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5639 pMixedCtx->SysEnter.esp = u64Val;
5640 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5641 }
5642 return rc;
5643}
5644
5645
5646/**
5647 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5648 * context.
5649 *
5650 * @returns VBox status code.
5651 * @param pVCpu Pointer to the VMCPU.
5652 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5653 * out-of-sync. Make sure to update the required fields
5654 * before using them.
5655 *
5656 * @remarks No-long-jump zone!!!
5657 */
5658static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5659{
5660 int rc = VINF_SUCCESS;
5661 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5662 {
5663 uint64_t u64Val = 0;
5664 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5665 pMixedCtx->fs.u64Base = u64Val;
5666 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5667 }
5668 return rc;
5669}
5670
5671
5672/**
5673 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5674 * context.
5675 *
5676 * @returns VBox status code.
5677 * @param pVCpu Pointer to the VMCPU.
5678 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5679 * out-of-sync. Make sure to update the required fields
5680 * before using them.
5681 *
5682 * @remarks No-long-jump zone!!!
5683 */
5684static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5685{
5686 int rc = VINF_SUCCESS;
5687 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5688 {
5689 uint64_t u64Val = 0;
5690 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5691 pMixedCtx->gs.u64Base = u64Val;
5692 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5693 }
5694 return rc;
5695}
5696
5697
5698/**
5699 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5700 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5701 * and TSC_AUX.
5702 *
5703 * @returns VBox status code.
5704 * @param pVCpu Pointer to the VMCPU.
5705 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5706 * out-of-sync. Make sure to update the required fields
5707 * before using them.
5708 *
5709 * @remarks No-long-jump zone!!!
5710 */
5711static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5712{
5713 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5714 return VINF_SUCCESS;
5715
5716#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5717 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5718 {
5719 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5720 pMsr += i;
5721 switch (pMsr->u32Msr)
5722 {
5723 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5724 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5725 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5726 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5727 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5728 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5729 default:
5730 {
5731 AssertFailed();
5732 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5733 }
5734 }
5735 }
5736#endif
5737
5738 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5739 return VINF_SUCCESS;
5740}
5741
5742
5743/**
5744 * Saves the guest control registers from the current VMCS into the guest-CPU
5745 * context.
5746 *
5747 * @returns VBox status code.
5748 * @param pVCpu Pointer to the VMCPU.
5749 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5750 * out-of-sync. Make sure to update the required fields
5751 * before using them.
5752 *
5753 * @remarks No-long-jump zone!!!
5754 */
5755static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5756{
5757 /* Guest CR0. Guest FPU. */
5758 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5759 AssertRCReturn(rc, rc);
5760
5761 /* Guest CR4. */
5762 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5763 AssertRCReturn(rc, rc);
5764
5765 /* Guest CR2 - updated always during the world-switch or in #PF. */
5766 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5767 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5768 {
5769 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5770 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5771
5772 PVM pVM = pVCpu->CTX_SUFF(pVM);
5773 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5774 || ( pVM->hm.s.fNestedPaging
5775 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5776 {
5777 uint64_t u64Val = 0;
5778 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5779 if (pMixedCtx->cr3 != u64Val)
5780 {
5781 CPUMSetGuestCR3(pVCpu, u64Val);
5782 if (VMMRZCallRing3IsEnabled(pVCpu))
5783 {
5784 PGMUpdateCR3(pVCpu, u64Val);
5785 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5786 }
5787 else
5788 {
5789 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5790 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5791 }
5792 }
5793
5794 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5795 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5796 {
5797 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5798 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5799 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5800 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5801
5802 if (VMMRZCallRing3IsEnabled(pVCpu))
5803 {
5804 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5805 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5806 }
5807 else
5808 {
5809 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5810 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5811 }
5812 }
5813 }
5814
5815 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5816 }
5817
5818 /*
5819 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5820 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5821 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5822 *
5823 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5824 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5825 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5826 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5827 *
5828 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5829 */
5830 if (VMMRZCallRing3IsEnabled(pVCpu))
5831 {
5832 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5833 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5834
5835 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5836 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5837
5838 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5839 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5840 }
5841
5842 return rc;
5843}
5844
5845
5846/**
5847 * Reads a guest segment register from the current VMCS into the guest-CPU
5848 * context.
5849 *
5850 * @returns VBox status code.
5851 * @param pVCpu Pointer to the VMCPU.
5852 * @param idxSel Index of the selector in the VMCS.
5853 * @param idxLimit Index of the segment limit in the VMCS.
5854 * @param idxBase Index of the segment base in the VMCS.
5855 * @param idxAccess Index of the access rights of the segment in the VMCS.
5856 * @param pSelReg Pointer to the segment selector.
5857 *
5858 * @remarks No-long-jump zone!!!
5859 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5860 * macro as that takes care of whether to read from the VMCS cache or
5861 * not.
5862 */
5863DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5864 PCPUMSELREG pSelReg)
5865{
5866 uint32_t u32Val = 0;
5867 int rc = VMXReadVmcs32(idxSel, &u32Val);
5868 AssertRCReturn(rc, rc);
5869 pSelReg->Sel = (uint16_t)u32Val;
5870 pSelReg->ValidSel = (uint16_t)u32Val;
5871 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5872
5873 rc = VMXReadVmcs32(idxLimit, &u32Val);
5874 AssertRCReturn(rc, rc);
5875 pSelReg->u32Limit = u32Val;
5876
5877 uint64_t u64Val = 0;
5878 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5879 AssertRCReturn(rc, rc);
5880 pSelReg->u64Base = u64Val;
5881
5882 rc = VMXReadVmcs32(idxAccess, &u32Val);
5883 AssertRCReturn(rc, rc);
5884 pSelReg->Attr.u = u32Val;
5885
5886 /*
5887 * If VT-x marks the segment as unusable, most other bits remain undefined:
5888 * - For CS the L, D and G bits have meaning.
5889 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5890 * - For the remaining data segments no bits are defined.
5891 *
5892 * The present bit and the unusable bit has been observed to be set at the
5893 * same time (the selector was supposed to invalid as we started executing
5894 * a V8086 interrupt in ring-0).
5895 *
5896 * What should be important for the rest of the VBox code that the P bit is
5897 * cleared. Some of the other VBox code recognizes the unusable bit, but
5898 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5899 * safe side here, we'll strip off P and other bits we don't care about. If
5900 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5901 *
5902 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5903 */
5904 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5905 {
5906 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5907
5908 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5909 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5910 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5911
5912 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5913#ifdef DEBUG_bird
5914 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5915 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5916 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5917#endif
5918 }
5919 return VINF_SUCCESS;
5920}
5921
5922
5923#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5924# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5925 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5926 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5927#else
5928# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5929 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5930 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5931#endif
5932
5933
5934/**
5935 * Saves the guest segment registers from the current VMCS into the guest-CPU
5936 * context.
5937 *
5938 * @returns VBox status code.
5939 * @param pVCpu Pointer to the VMCPU.
5940 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5941 * out-of-sync. Make sure to update the required fields
5942 * before using them.
5943 *
5944 * @remarks No-long-jump zone!!!
5945 */
5946static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5947{
5948 /* Guest segment registers. */
5949 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5950 {
5951 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5952 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5953 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5954 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5955 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5956 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5957 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5958
5959 /* Restore segment attributes for real-on-v86 mode hack. */
5960 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5961 {
5962 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5963 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5964 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5965 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5966 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5967 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5968 }
5969 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5970 }
5971
5972 return VINF_SUCCESS;
5973}
5974
5975
5976/**
5977 * Saves the guest descriptor table registers and task register from the current
5978 * VMCS into the guest-CPU context.
5979 *
5980 * @returns VBox status code.
5981 * @param pVCpu Pointer to the VMCPU.
5982 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5983 * out-of-sync. Make sure to update the required fields
5984 * before using them.
5985 *
5986 * @remarks No-long-jump zone!!!
5987 */
5988static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5989{
5990 int rc = VINF_SUCCESS;
5991
5992 /* Guest LDTR. */
5993 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5994 {
5995 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5996 AssertRCReturn(rc, rc);
5997 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5998 }
5999
6000 /* Guest GDTR. */
6001 uint64_t u64Val = 0;
6002 uint32_t u32Val = 0;
6003 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6004 {
6005 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6006 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6007 pMixedCtx->gdtr.pGdt = u64Val;
6008 pMixedCtx->gdtr.cbGdt = u32Val;
6009 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
6010 }
6011
6012 /* Guest IDTR. */
6013 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6014 {
6015 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6016 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6017 pMixedCtx->idtr.pIdt = u64Val;
6018 pMixedCtx->idtr.cbIdt = u32Val;
6019 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
6020 }
6021
6022 /* Guest TR. */
6023 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6024 {
6025 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6026 AssertRCReturn(rc, rc);
6027
6028 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6029 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6030 {
6031 rc = VMXLOCAL_READ_SEG(TR, tr);
6032 AssertRCReturn(rc, rc);
6033 }
6034 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
6035 }
6036 return rc;
6037}
6038
6039#undef VMXLOCAL_READ_SEG
6040
6041
6042/**
6043 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6044 * context.
6045 *
6046 * @returns VBox status code.
6047 * @param pVCpu Pointer to the VMCPU.
6048 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6049 * out-of-sync. Make sure to update the required fields
6050 * before using them.
6051 *
6052 * @remarks No-long-jump zone!!!
6053 */
6054static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6055{
6056 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6057 {
6058 if (!pVCpu->hm.s.fUsingHyperDR7)
6059 {
6060 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6061 uint32_t u32Val;
6062 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6063 pMixedCtx->dr[7] = u32Val;
6064 }
6065
6066 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
6067 }
6068 return VINF_SUCCESS;
6069}
6070
6071
6072/**
6073 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6074 *
6075 * @returns VBox status code.
6076 * @param pVCpu Pointer to the VMCPU.
6077 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6078 * out-of-sync. Make sure to update the required fields
6079 * before using them.
6080 *
6081 * @remarks No-long-jump zone!!!
6082 */
6083static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6084{
6085 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6086 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
6087 return VINF_SUCCESS;
6088}
6089
6090
6091/**
6092 * Saves the entire guest state from the currently active VMCS into the
6093 * guest-CPU context. This essentially VMREADs all guest-data.
6094 *
6095 * @returns VBox status code.
6096 * @param pVCpu Pointer to the VMCPU.
6097 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6098 * out-of-sync. Make sure to update the required fields
6099 * before using them.
6100 */
6101static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6102{
6103 Assert(pVCpu);
6104 Assert(pMixedCtx);
6105
6106 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6107 return VINF_SUCCESS;
6108
6109 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6110 again on the ring-3 callback path, there is no real need to. */
6111 if (VMMRZCallRing3IsEnabled(pVCpu))
6112 VMMR0LogFlushDisable(pVCpu);
6113 else
6114 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6115 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6116
6117 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6118 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6119
6120 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6121 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6122
6123 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6124 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6125
6126 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6127 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6128
6129 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6130 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6131
6132 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6133 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6134
6135 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
6136 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6137
6138 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
6139 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6140
6141 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6142 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6143
6144 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6145 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6146
6147 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6148 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6149
6150 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6151 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6152
6153 if (VMMRZCallRing3IsEnabled(pVCpu))
6154 VMMR0LogFlushEnable(pVCpu);
6155
6156 return rc;
6157}
6158
6159
6160/**
6161 * Check per-VM and per-VCPU force flag actions that require us to go back to
6162 * ring-3 for one reason or another.
6163 *
6164 * @returns VBox status code (information status code included).
6165 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6166 * ring-3.
6167 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6168 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6169 * interrupts)
6170 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6171 * all EMTs to be in ring-3.
6172 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6173 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6174 * to the EM loop.
6175 *
6176 * @param pVM Pointer to the VM.
6177 * @param pVCpu Pointer to the VMCPU.
6178 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6179 * out-of-sync. Make sure to update the required fields
6180 * before using them.
6181 */
6182static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6183{
6184 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6185
6186 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6187 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6188 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6189 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6190 {
6191 /* We need the control registers now, make sure the guest-CPU context is updated. */
6192 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6193 AssertRCReturn(rc3, rc3);
6194
6195 /* Pending HM CR3 sync. */
6196 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6197 {
6198 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6199 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6200 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6201 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6202 }
6203
6204 /* Pending HM PAE PDPEs. */
6205 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6206 {
6207 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6208 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6209 }
6210
6211 /* Pending PGM C3 sync. */
6212 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6213 {
6214 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6215 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6216 if (rc2 != VINF_SUCCESS)
6217 {
6218 AssertRC(rc2);
6219 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6220 return rc2;
6221 }
6222 }
6223
6224 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6225 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6226 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6227 {
6228 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6229 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6230 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6231 return rc2;
6232 }
6233
6234 /* Pending VM request packets, such as hardware interrupts. */
6235 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6236 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6237 {
6238 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6239 return VINF_EM_PENDING_REQUEST;
6240 }
6241
6242 /* Pending PGM pool flushes. */
6243 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6244 {
6245 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6246 return VINF_PGM_POOL_FLUSH_PENDING;
6247 }
6248
6249 /* Pending DMA requests. */
6250 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6251 {
6252 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6253 return VINF_EM_RAW_TO_R3;
6254 }
6255 }
6256
6257 return VINF_SUCCESS;
6258}
6259
6260
6261/**
6262 * Converts any TRPM trap into a pending HM event. This is typically used when
6263 * entering from ring-3 (not longjmp returns).
6264 *
6265 * @param pVCpu Pointer to the VMCPU.
6266 */
6267static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6268{
6269 Assert(TRPMHasTrap(pVCpu));
6270 Assert(!pVCpu->hm.s.Event.fPending);
6271
6272 uint8_t uVector;
6273 TRPMEVENT enmTrpmEvent;
6274 RTGCUINT uErrCode;
6275 RTGCUINTPTR GCPtrFaultAddress;
6276 uint8_t cbInstr;
6277
6278 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6279 AssertRC(rc);
6280
6281 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6282 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6283 if (enmTrpmEvent == TRPM_TRAP)
6284 {
6285 switch (uVector)
6286 {
6287 case X86_XCPT_BP:
6288 case X86_XCPT_OF:
6289 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6290 break;
6291
6292 case X86_XCPT_PF:
6293 case X86_XCPT_DF:
6294 case X86_XCPT_TS:
6295 case X86_XCPT_NP:
6296 case X86_XCPT_SS:
6297 case X86_XCPT_GP:
6298 case X86_XCPT_AC:
6299 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6300 /* no break! */
6301 default:
6302 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6303 break;
6304 }
6305 }
6306 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6307 {
6308 if (uVector == X86_XCPT_NMI)
6309 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6310 else
6311 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6312 }
6313 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6314 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6315 else
6316 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6317
6318 rc = TRPMResetTrap(pVCpu);
6319 AssertRC(rc);
6320 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6321 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6322
6323 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6324 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6325}
6326
6327
6328/**
6329 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6330 * VT-x to execute any instruction.
6331 *
6332 * @param pvCpu Pointer to the VMCPU.
6333 */
6334static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6335{
6336 Assert(pVCpu->hm.s.Event.fPending);
6337
6338 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6339 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6340 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6341 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6342
6343 /* If a trap was already pending, we did something wrong! */
6344 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6345
6346 TRPMEVENT enmTrapType;
6347 switch (uVectorType)
6348 {
6349 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6350 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6351 enmTrapType = TRPM_HARDWARE_INT;
6352 break;
6353
6354 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6355 enmTrapType = TRPM_SOFTWARE_INT;
6356 break;
6357
6358 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6359 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6360 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6361 enmTrapType = TRPM_TRAP;
6362 break;
6363
6364 default:
6365 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6366 enmTrapType = TRPM_32BIT_HACK;
6367 break;
6368 }
6369
6370 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6371
6372 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6373 AssertRC(rc);
6374
6375 if (fErrorCodeValid)
6376 TRPMSetErrorCode(pVCpu, uErrorCode);
6377
6378 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6379 && uVector == X86_XCPT_PF)
6380 {
6381 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6382 }
6383 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6384 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6385 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6386 {
6387 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6388 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6389 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6390 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6391 }
6392 pVCpu->hm.s.Event.fPending = false;
6393}
6394
6395
6396/**
6397 * Does the necessary state syncing before returning to ring-3 for any reason
6398 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6399 *
6400 * @returns VBox status code.
6401 * @param pVM Pointer to the VM.
6402 * @param pVCpu Pointer to the VMCPU.
6403 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6404 * be out-of-sync. Make sure to update the required
6405 * fields before using them.
6406 * @param fSaveGuestState Whether to save the guest state or not.
6407 *
6408 * @remarks If you modify code here, make sure to check whether
6409 * hmR0VmxCallRing3Callback() needs to be updated too.
6410 * @remarks No-long-jmp zone!!!
6411 */
6412static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6413{
6414 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6415 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6416
6417 RTCPUID idCpu = RTMpCpuId();
6418 Log4Func(("HostCpuId=%u\n", idCpu));
6419
6420 /* Save the guest state if necessary. */
6421 if ( fSaveGuestState
6422 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6423 {
6424 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6425 AssertRCReturn(rc, rc);
6426 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6427 }
6428
6429 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6430 if (CPUMIsGuestFPUStateActive(pVCpu))
6431 {
6432 /* We shouldn't reload CR0 without saving it first. */
6433 if (!fSaveGuestState)
6434 {
6435 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6436 AssertRCReturn(rc, rc);
6437 }
6438 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6439 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6440 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6441 }
6442
6443 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6444#ifdef VBOX_STRICT
6445 if (CPUMIsHyperDebugStateActive(pVCpu))
6446 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6447#endif
6448 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6449 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6450 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6451 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6452
6453#if HC_ARCH_BITS == 64
6454 /* Restore host-state bits that VT-x only restores partially. */
6455 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6456 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6457 {
6458 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6459 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6460 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6461 }
6462#endif
6463
6464 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6465 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6466 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6467 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6468 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6469 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6470 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6471 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6472
6473 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6474
6475 /** @todo This kinda defeats the purpose of having preemption hooks.
6476 * The problem is, deregistering the hooks should be moved to a place that
6477 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6478 * context.
6479 */
6480 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6481 {
6482 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6483 AssertRCReturn(rc, rc);
6484
6485 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6486 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6487 }
6488 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6489 NOREF(idCpu);
6490
6491 return VINF_SUCCESS;
6492}
6493
6494
6495/**
6496 * Leaves the VT-x session.
6497 *
6498 * @returns VBox status code.
6499 * @param pVM Pointer to the VM.
6500 * @param pVCpu Pointer to the VMCPU.
6501 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6502 * out-of-sync. Make sure to update the required fields
6503 * before using them.
6504 *
6505 * @remarks No-long-jmp zone!!!
6506 */
6507DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6508{
6509 HM_DISABLE_PREEMPT_IF_NEEDED();
6510 HMVMX_ASSERT_CPU_SAFE();
6511 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6512 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6513
6514 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6515 and done this from the VMXR0ThreadCtxCallback(). */
6516 if (!pVCpu->hm.s.fLeaveDone)
6517 {
6518 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6519 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6520 pVCpu->hm.s.fLeaveDone = true;
6521 }
6522
6523 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6524 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6525 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6526 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6527 VMMR0ThreadCtxHooksDeregister(pVCpu);
6528
6529 /* Leave HM context. This takes care of local init (term). */
6530 int rc = HMR0LeaveCpu(pVCpu);
6531
6532 HM_RESTORE_PREEMPT_IF_NEEDED();
6533
6534 return rc;
6535}
6536
6537
6538/**
6539 * Does the necessary state syncing before doing a longjmp to ring-3.
6540 *
6541 * @returns VBox status code.
6542 * @param pVM Pointer to the VM.
6543 * @param pVCpu Pointer to the VMCPU.
6544 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6545 * out-of-sync. Make sure to update the required fields
6546 * before using them.
6547 *
6548 * @remarks No-long-jmp zone!!!
6549 */
6550DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6551{
6552 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6553}
6554
6555
6556/**
6557 * Take necessary actions before going back to ring-3.
6558 *
6559 * An action requires us to go back to ring-3. This function does the necessary
6560 * steps before we can safely return to ring-3. This is not the same as longjmps
6561 * to ring-3, this is voluntary and prepares the guest so it may continue
6562 * executing outside HM (recompiler/IEM).
6563 *
6564 * @returns VBox status code.
6565 * @param pVM Pointer to the VM.
6566 * @param pVCpu Pointer to the VMCPU.
6567 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6568 * out-of-sync. Make sure to update the required fields
6569 * before using them.
6570 * @param rcExit The reason for exiting to ring-3. Can be
6571 * VINF_VMM_UNKNOWN_RING3_CALL.
6572 */
6573static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6574{
6575 Assert(pVM);
6576 Assert(pVCpu);
6577 Assert(pMixedCtx);
6578 HMVMX_ASSERT_PREEMPT_SAFE();
6579
6580 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6581 {
6582 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6583 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6584 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6585 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6586 }
6587
6588 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6589 VMMRZCallRing3Disable(pVCpu);
6590 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6591
6592 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6593 if (pVCpu->hm.s.Event.fPending)
6594 {
6595 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6596 Assert(!pVCpu->hm.s.Event.fPending);
6597 }
6598
6599 /* Save guest state and restore host state bits. */
6600 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6601 AssertRCReturn(rc, rc);
6602 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6603
6604 /* Sync recompiler state. */
6605 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6606 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6607 | CPUM_CHANGED_LDTR
6608 | CPUM_CHANGED_GDTR
6609 | CPUM_CHANGED_IDTR
6610 | CPUM_CHANGED_TR
6611 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6612 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6613 if ( pVM->hm.s.fNestedPaging
6614 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6615 {
6616 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6617 }
6618
6619 Assert(!pVCpu->hm.s.fClearTrapFlag);
6620
6621 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6622 if (rcExit != VINF_EM_RAW_INTERRUPT)
6623 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6624
6625 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6626
6627 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6628 VMMRZCallRing3RemoveNotification(pVCpu);
6629 VMMRZCallRing3Enable(pVCpu);
6630
6631 return rc;
6632}
6633
6634
6635/**
6636 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6637 * longjump to ring-3 and possibly get preempted.
6638 *
6639 * @returns VBox status code.
6640 * @param pVCpu Pointer to the VMCPU.
6641 * @param enmOperation The operation causing the ring-3 longjump.
6642 * @param pvUser Opaque pointer to the guest-CPU context. The data
6643 * may be out-of-sync. Make sure to update the required
6644 * fields before using them.
6645 * @remarks If you modify code here, make sure to check whether
6646 * hmR0VmxLeave() needs to be updated too.
6647 */
6648DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6649{
6650 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6651 {
6652 VMMRZCallRing3RemoveNotification(pVCpu);
6653 HM_DISABLE_PREEMPT_IF_NEEDED();
6654
6655 /* If anything here asserts or fails, good luck. */
6656 if (CPUMIsGuestFPUStateActive(pVCpu))
6657 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6658
6659 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6660
6661#if HC_ARCH_BITS == 64
6662 /* Restore host-state bits that VT-x only restores partially. */
6663 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6664 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6665 {
6666 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6667 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6668 }
6669#endif
6670 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6671 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6672 {
6673 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6674 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6675 }
6676
6677 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6678 VMMR0ThreadCtxHooksDeregister(pVCpu);
6679
6680 HMR0LeaveCpu(pVCpu);
6681 HM_RESTORE_PREEMPT_IF_NEEDED();
6682 return VINF_SUCCESS;
6683 }
6684
6685 Assert(pVCpu);
6686 Assert(pvUser);
6687 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6688 HMVMX_ASSERT_PREEMPT_SAFE();
6689
6690 VMMRZCallRing3Disable(pVCpu);
6691 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6692
6693 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6694 enmOperation));
6695
6696 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6697 AssertRCReturn(rc, rc);
6698
6699 VMMRZCallRing3Enable(pVCpu);
6700 return VINF_SUCCESS;
6701}
6702
6703
6704/**
6705 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6706 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6707 *
6708 * @param pVCpu Pointer to the VMCPU.
6709 */
6710DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6711{
6712 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6713 {
6714 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6715 {
6716 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6717 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6718 AssertRC(rc);
6719 }
6720 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6721}
6722
6723
6724/**
6725 * Evaluates the event to be delivered to the guest and sets it as the pending
6726 * event.
6727 *
6728 * @param pVCpu Pointer to the VMCPU.
6729 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6730 * out-of-sync. Make sure to update the required fields
6731 * before using them.
6732 */
6733static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6734{
6735 Assert(!pVCpu->hm.s.Event.fPending);
6736
6737 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6738 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6739 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6740 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6741
6742 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6743 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6744 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6745 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6746 Assert(!TRPMHasTrap(pVCpu));
6747
6748 /** @todo SMI. SMIs take priority over NMIs. */
6749 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6750 {
6751 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6752 if ( !fBlockMovSS
6753 && !fBlockSti)
6754 {
6755 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6756 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6757 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6758 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6759
6760 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6761 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6762 }
6763 else
6764 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6765 }
6766 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6767 && !pVCpu->hm.s.fSingleInstruction)
6768 {
6769 /*
6770 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6771 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6772 * evaluated here and not set as pending, solely based on the force-flags.
6773 */
6774 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6775 AssertRC(rc);
6776 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6777 if ( !fBlockInt
6778 && !fBlockSti
6779 && !fBlockMovSS)
6780 {
6781 uint8_t u8Interrupt;
6782 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6783 if (RT_SUCCESS(rc))
6784 {
6785 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6786 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6787 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6788
6789 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6790 }
6791 else
6792 {
6793 /** @todo Does this actually happen? If not turn it into an assertion. */
6794 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6795 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6796 }
6797 }
6798 else
6799 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6800 }
6801}
6802
6803
6804/**
6805 * Injects any pending events into the guest if the guest is in a state to
6806 * receive them.
6807 *
6808 * @returns VBox status code (informational status codes included).
6809 * @param pVCpu Pointer to the VMCPU.
6810 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6811 * out-of-sync. Make sure to update the required fields
6812 * before using them.
6813 */
6814static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6815{
6816 HMVMX_ASSERT_PREEMPT_SAFE();
6817 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6818
6819 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6820 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6821 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6822 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6823
6824 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6825 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6826 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6827 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6828 Assert(!TRPMHasTrap(pVCpu));
6829
6830 int rc = VINF_SUCCESS;
6831 if (pVCpu->hm.s.Event.fPending)
6832 {
6833#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6834 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6835 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6836 {
6837 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6838 AssertRCReturn(rc, rc);
6839 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6840 Assert(!fBlockInt);
6841 Assert(!fBlockSti);
6842 Assert(!fBlockMovSS);
6843 }
6844 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6845 {
6846 Assert(!fBlockSti);
6847 Assert(!fBlockMovSS);
6848 }
6849#endif
6850 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6851 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6852 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6853 AssertRCReturn(rc, rc);
6854
6855 /* Update the interruptibility-state as it could have been changed by
6856 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6857 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6858 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6859
6860#ifdef VBOX_WITH_STATISTICS
6861 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6862 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6863 else
6864 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6865#endif
6866 }
6867
6868 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6869 int rc2 = VINF_SUCCESS;
6870 if ( fBlockSti
6871 || fBlockMovSS)
6872 {
6873 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6874 {
6875 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6876 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6877 {
6878 /*
6879 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6880 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6881 * See Intel spec. 27.3.4 "Saving Non-Register State".
6882 */
6883 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6884 AssertRCReturn(rc, rc);
6885 }
6886 }
6887 else
6888 {
6889 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6890 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6891 uIntrState = 0;
6892 }
6893 }
6894
6895 /*
6896 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6897 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6898 */
6899 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6900 AssertRC(rc2);
6901
6902 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6903 return rc;
6904}
6905
6906
6907/**
6908 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6909 *
6910 * @param pVCpu Pointer to the VMCPU.
6911 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6912 * out-of-sync. Make sure to update the required fields
6913 * before using them.
6914 */
6915DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6916{
6917 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6918 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6919}
6920
6921
6922/**
6923 * Injects a double-fault (#DF) exception into the VM.
6924 *
6925 * @returns VBox status code (informational status code included).
6926 * @param pVCpu Pointer to the VMCPU.
6927 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6928 * out-of-sync. Make sure to update the required fields
6929 * before using them.
6930 */
6931DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6932{
6933 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6934 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6935 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6936 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6937 puIntrState);
6938}
6939
6940
6941/**
6942 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6943 *
6944 * @param pVCpu Pointer to the VMCPU.
6945 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6946 * out-of-sync. Make sure to update the required fields
6947 * before using them.
6948 */
6949DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6950{
6951 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6952 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6953 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6954}
6955
6956
6957/**
6958 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6959 *
6960 * @param pVCpu Pointer to the VMCPU.
6961 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6962 * out-of-sync. Make sure to update the required fields
6963 * before using them.
6964 * @param cbInstr The value of RIP that is to be pushed on the guest
6965 * stack.
6966 */
6967DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6968{
6969 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6970 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6971 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6972}
6973
6974
6975/**
6976 * Injects a general-protection (#GP) fault into the VM.
6977 *
6978 * @returns VBox status code (informational status code included).
6979 * @param pVCpu Pointer to the VMCPU.
6980 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6981 * out-of-sync. Make sure to update the required fields
6982 * before using them.
6983 * @param u32ErrorCode The error code associated with the #GP.
6984 */
6985DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6986 uint32_t *puIntrState)
6987{
6988 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6989 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6990 if (fErrorCodeValid)
6991 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6992 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6993 puIntrState);
6994}
6995
6996
6997/**
6998 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6999 *
7000 * @param pVCpu Pointer to the VMCPU.
7001 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7002 * out-of-sync. Make sure to update the required fields
7003 * before using them.
7004 * @param uVector The software interrupt vector number.
7005 * @param cbInstr The value of RIP that is to be pushed on the guest
7006 * stack.
7007 */
7008DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7009{
7010 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7011 if ( uVector == X86_XCPT_BP
7012 || uVector == X86_XCPT_OF)
7013 {
7014 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7015 }
7016 else
7017 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7018 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7019}
7020
7021
7022/**
7023 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7024 * stack.
7025 *
7026 * @returns VBox status code (information status code included).
7027 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7028 * @param pVM Pointer to the VM.
7029 * @param pMixedCtx Pointer to the guest-CPU context.
7030 * @param uValue The value to push to the guest stack.
7031 */
7032DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7033{
7034 /*
7035 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7036 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7037 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7038 */
7039 if (pMixedCtx->sp == 1)
7040 return VINF_EM_RESET;
7041 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7042 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7043 AssertRCReturn(rc, rc);
7044 return rc;
7045}
7046
7047
7048/**
7049 * Injects an event into the guest upon VM-entry by updating the relevant fields
7050 * in the VM-entry area in the VMCS.
7051 *
7052 * @returns VBox status code (informational error codes included).
7053 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7054 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7055 *
7056 * @param pVCpu Pointer to the VMCPU.
7057 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7058 * be out-of-sync. Make sure to update the required
7059 * fields before using them.
7060 * @param u64IntInfo The VM-entry interruption-information field.
7061 * @param cbInstr The VM-entry instruction length in bytes (for
7062 * software interrupts, exceptions and privileged
7063 * software exceptions).
7064 * @param u32ErrCode The VM-entry exception error code.
7065 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7066 * @param puIntrState Pointer to the current guest interruptibility-state.
7067 * This interruptibility-state will be updated if
7068 * necessary. This cannot not be NULL.
7069 *
7070 * @remarks Requires CR0!
7071 * @remarks No-long-jump zone!!!
7072 */
7073static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7074 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7075{
7076 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7077 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7078 Assert(puIntrState);
7079 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7080
7081 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7082 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7083
7084#ifdef VBOX_STRICT
7085 /* Validate the error-code-valid bit for hardware exceptions. */
7086 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7087 {
7088 switch (uVector)
7089 {
7090 case X86_XCPT_PF:
7091 case X86_XCPT_DF:
7092 case X86_XCPT_TS:
7093 case X86_XCPT_NP:
7094 case X86_XCPT_SS:
7095 case X86_XCPT_GP:
7096 case X86_XCPT_AC:
7097 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7098 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7099 /* fallthru */
7100 default:
7101 break;
7102 }
7103 }
7104#endif
7105
7106 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7107 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7108 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7109
7110 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7111
7112 /* We require CR0 to check if the guest is in real-mode. */
7113 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7114 AssertRCReturn(rc, rc);
7115
7116 /*
7117 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7118 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7119 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7120 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7121 */
7122 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7123 {
7124 PVM pVM = pVCpu->CTX_SUFF(pVM);
7125 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7126 {
7127 Assert(PDMVmmDevHeapIsEnabled(pVM));
7128 Assert(pVM->hm.s.vmx.pRealModeTSS);
7129
7130 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7131 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7132 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7133 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7134 AssertRCReturn(rc, rc);
7135 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7136
7137 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7138 const size_t cbIdtEntry = sizeof(X86IDTR16);
7139 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7140 {
7141 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7142 if (uVector == X86_XCPT_DF)
7143 return VINF_EM_RESET;
7144 else if (uVector == X86_XCPT_GP)
7145 {
7146 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7147 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7148 }
7149
7150 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7151 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7152 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7153 }
7154
7155 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7156 uint16_t uGuestIp = pMixedCtx->ip;
7157 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7158 {
7159 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7160 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7161 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7162 }
7163 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7164 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7165
7166 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7167 X86IDTR16 IdtEntry;
7168 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7169 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7170 AssertRCReturn(rc, rc);
7171
7172 /* Construct the stack frame for the interrupt/exception handler. */
7173 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7174 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7175 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7176 AssertRCReturn(rc, rc);
7177
7178 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7179 if (rc == VINF_SUCCESS)
7180 {
7181 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7182 pMixedCtx->rip = IdtEntry.offSel;
7183 pMixedCtx->cs.Sel = IdtEntry.uSel;
7184 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7185 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7186 && uVector == X86_XCPT_PF)
7187 {
7188 pMixedCtx->cr2 = GCPtrFaultAddress;
7189 }
7190
7191 /* If any other guest-state bits are changed here, make sure to update
7192 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7193 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7194 | HM_CHANGED_GUEST_RIP
7195 | HM_CHANGED_GUEST_RFLAGS
7196 | HM_CHANGED_GUEST_RSP);
7197
7198 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7199 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7200 {
7201 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7202 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7203 Log4(("Clearing inhibition due to STI.\n"));
7204 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7205 }
7206 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7207
7208 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7209 it, if we are returning to ring-3 before executing guest code. */
7210 pVCpu->hm.s.Event.fPending = false;
7211 }
7212 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7213 return rc;
7214 }
7215 else
7216 {
7217 /*
7218 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7219 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7220 */
7221 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7222 }
7223 }
7224
7225 /* Validate. */
7226 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7227 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7228 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7229
7230 /* Inject. */
7231 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7232 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7233 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7234 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7235
7236 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7237 && uVector == X86_XCPT_PF)
7238 {
7239 pMixedCtx->cr2 = GCPtrFaultAddress;
7240 }
7241
7242 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7243 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7244
7245 AssertRCReturn(rc, rc);
7246 return rc;
7247}
7248
7249
7250/**
7251 * Clears the interrupt-window exiting control in the VMCS and if necessary
7252 * clears the current event in the VMCS as well.
7253 *
7254 * @returns VBox status code.
7255 * @param pVCpu Pointer to the VMCPU.
7256 *
7257 * @remarks Use this function only to clear events that have not yet been
7258 * delivered to the guest but are injected in the VMCS!
7259 * @remarks No-long-jump zone!!!
7260 */
7261static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7262{
7263 int rc;
7264 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7265
7266 /* Clear interrupt-window exiting control. */
7267 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7268 {
7269 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7270 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7271 AssertRC(rc);
7272 }
7273
7274 if (!pVCpu->hm.s.Event.fPending)
7275 return;
7276
7277#ifdef VBOX_STRICT
7278 uint32_t u32EntryInfo;
7279 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7280 AssertRC(rc);
7281 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7282#endif
7283
7284 /* Clear the entry-interruption field (including the valid bit). */
7285 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7286 AssertRC(rc);
7287
7288 /* Clear the pending debug exception field. */
7289 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7290 AssertRC(rc);
7291}
7292
7293
7294/**
7295 * Enters the VT-x session.
7296 *
7297 * @returns VBox status code.
7298 * @param pVM Pointer to the VM.
7299 * @param pVCpu Pointer to the VMCPU.
7300 * @param pCpu Pointer to the CPU info struct.
7301 */
7302VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7303{
7304 AssertPtr(pVM);
7305 AssertPtr(pVCpu);
7306 Assert(pVM->hm.s.vmx.fSupported);
7307 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7308 NOREF(pCpu);
7309
7310 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7311 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7312
7313#ifdef VBOX_STRICT
7314 /* Make sure we're in VMX root mode. */
7315 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7316 if (!(u32HostCR4 & X86_CR4_VMXE))
7317 {
7318 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7319 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7320 }
7321#endif
7322
7323 /*
7324 * Load the VCPU's VMCS as the current (and active) one.
7325 */
7326 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7327 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7328 if (RT_FAILURE(rc))
7329 return rc;
7330
7331 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7332 pVCpu->hm.s.fLeaveDone = false;
7333 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7334
7335 return VINF_SUCCESS;
7336}
7337
7338
7339/**
7340 * The thread-context callback (only on platforms which support it).
7341 *
7342 * @param enmEvent The thread-context event.
7343 * @param pVCpu Pointer to the VMCPU.
7344 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7345 * @thread EMT(pVCpu)
7346 */
7347VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7348{
7349 switch (enmEvent)
7350 {
7351 case RTTHREADCTXEVENT_PREEMPTING:
7352 {
7353 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7354 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7355 VMCPU_ASSERT_EMT(pVCpu);
7356
7357 PVM pVM = pVCpu->CTX_SUFF(pVM);
7358 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7359
7360 /* No longjmps (logger flushes, locks) in this fragile context. */
7361 VMMRZCallRing3Disable(pVCpu);
7362 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7363
7364 /*
7365 * Restore host-state (FPU, debug etc.)
7366 */
7367 if (!pVCpu->hm.s.fLeaveDone)
7368 {
7369 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7370 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7371 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7372 pVCpu->hm.s.fLeaveDone = true;
7373 }
7374
7375 /* Leave HM context, takes care of local init (term). */
7376 int rc = HMR0LeaveCpu(pVCpu);
7377 AssertRC(rc); NOREF(rc);
7378
7379 /* Restore longjmp state. */
7380 VMMRZCallRing3Enable(pVCpu);
7381 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7382 break;
7383 }
7384
7385 case RTTHREADCTXEVENT_RESUMED:
7386 {
7387 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7388 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7389 VMCPU_ASSERT_EMT(pVCpu);
7390
7391 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7392 VMMRZCallRing3Disable(pVCpu);
7393 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7394
7395 /* Initialize the bare minimum state required for HM. This takes care of
7396 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7397 int rc = HMR0EnterCpu(pVCpu);
7398 AssertRC(rc);
7399 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7400
7401 /* Load the active VMCS as the current one. */
7402 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7403 {
7404 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7405 AssertRC(rc); NOREF(rc);
7406 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7407 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7408 }
7409 pVCpu->hm.s.fLeaveDone = false;
7410
7411 /* Restore longjmp state. */
7412 VMMRZCallRing3Enable(pVCpu);
7413 break;
7414 }
7415
7416 default:
7417 break;
7418 }
7419}
7420
7421
7422/**
7423 * Saves the host state in the VMCS host-state.
7424 * Sets up the VM-exit MSR-load area.
7425 *
7426 * The CPU state will be loaded from these fields on every successful VM-exit.
7427 *
7428 * @returns VBox status code.
7429 * @param pVM Pointer to the VM.
7430 * @param pVCpu Pointer to the VMCPU.
7431 *
7432 * @remarks No-long-jump zone!!!
7433 */
7434static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7435{
7436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7437
7438 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7439 return VINF_SUCCESS;
7440
7441 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7442 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7443
7444 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7445 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7446
7447 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7448 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7449
7450 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7451 return rc;
7452}
7453
7454
7455/**
7456 * Saves the host state in the VMCS host-state.
7457 *
7458 * @returns VBox status code.
7459 * @param pVM Pointer to the VM.
7460 * @param pVCpu Pointer to the VMCPU.
7461 *
7462 * @remarks No-long-jump zone!!!
7463 */
7464VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7465{
7466 AssertPtr(pVM);
7467 AssertPtr(pVCpu);
7468
7469 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7470
7471 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7472 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7473 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7474 return hmR0VmxSaveHostState(pVM, pVCpu);
7475}
7476
7477
7478/**
7479 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7480 * loaded from these fields on every successful VM-entry.
7481 *
7482 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7483 * Sets up the VM-entry controls.
7484 * Sets up the appropriate VMX non-root function to execute guest code based on
7485 * the guest CPU mode.
7486 *
7487 * @returns VBox status code.
7488 * @param pVM Pointer to the VM.
7489 * @param pVCpu Pointer to the VMCPU.
7490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7491 * out-of-sync. Make sure to update the required fields
7492 * before using them.
7493 *
7494 * @remarks No-long-jump zone!!!
7495 */
7496static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7497{
7498 AssertPtr(pVM);
7499 AssertPtr(pVCpu);
7500 AssertPtr(pMixedCtx);
7501 HMVMX_ASSERT_PREEMPT_SAFE();
7502
7503#ifdef LOG_ENABLED
7504 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7505 * probably not initialized yet? Anyway this will do for now.
7506 *
7507 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7508 * interface and disable ring-3 calls when thread-context hooks are not
7509 * available. */
7510 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7511 VMMR0LogFlushDisable(pVCpu);
7512#endif
7513
7514 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7515
7516 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7517
7518 /* Determine real-on-v86 mode. */
7519 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7520 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7521 && CPUMIsGuestInRealModeEx(pMixedCtx))
7522 {
7523 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7524 }
7525
7526 /*
7527 * Load the guest-state into the VMCS.
7528 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7529 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7530 */
7531 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7532 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7533
7534 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7535 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7536 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7537
7538 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7539 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7540 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7541
7542 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7543 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7544
7545 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7546 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7547
7548 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7549 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7550 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7551
7552 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7553 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7554
7555 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7556 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7557
7558 /*
7559 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7560 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7561 */
7562 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7563 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7564
7565 /* Clear any unused and reserved bits. */
7566 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7567
7568#ifdef LOG_ENABLED
7569 /* Only reenable log-flushing if the caller has it enabled. */
7570 if (!fCallerDisabledLogFlush)
7571 VMMR0LogFlushEnable(pVCpu);
7572#endif
7573
7574 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7575 return rc;
7576}
7577
7578
7579/**
7580 * Loads the state shared between the host and guest into the VMCS.
7581 *
7582 * @param pVM Pointer to the VM.
7583 * @param pVCpu Pointer to the VMCPU.
7584 * @param pCtx Pointer to the guest-CPU context.
7585 *
7586 * @remarks No-long-jump zone!!!
7587 */
7588static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7589{
7590 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7591 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7592
7593 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7594 {
7595 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7596 AssertRC(rc);
7597 }
7598
7599 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7600 {
7601 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7602 AssertRC(rc);
7603
7604 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7605 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7606 {
7607 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7608 AssertRC(rc);
7609 }
7610 }
7611
7612 AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7613 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7614}
7615
7616
7617/**
7618 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7619 *
7620 * @param pVM Pointer to the VM.
7621 * @param pVCpu Pointer to the VMCPU.
7622 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7623 * out-of-sync. Make sure to update the required fields
7624 * before using them.
7625 */
7626DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7627{
7628 HMVMX_ASSERT_PREEMPT_SAFE();
7629
7630 Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7631#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7632 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7633#endif
7634
7635 if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7636 {
7637 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7638 AssertRC(rc);
7639 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7640 }
7641 else if (VMCPU_HMCF_VALUE(pVCpu))
7642 {
7643 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7644 AssertRC(rc);
7645 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7646 }
7647
7648 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7649 AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7650 || VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
7651 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7652
7653#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7654 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7655 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7656 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7657#endif
7658}
7659
7660
7661/**
7662 * Does the preparations before executing guest code in VT-x.
7663 *
7664 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7665 * recompiler. We must be cautious what we do here regarding committing
7666 * guest-state information into the VMCS assuming we assuredly execute the
7667 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7668 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7669 * so that the recompiler can (and should) use them when it resumes guest
7670 * execution. Otherwise such operations must be done when we can no longer
7671 * exit to ring-3.
7672 *
7673 * @returns Strict VBox status code.
7674 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7675 * have been disabled.
7676 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7677 * double-fault into the guest.
7678 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7679 *
7680 * @param pVM Pointer to the VM.
7681 * @param pVCpu Pointer to the VMCPU.
7682 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7683 * out-of-sync. Make sure to update the required fields
7684 * before using them.
7685 * @param pVmxTransient Pointer to the VMX transient structure.
7686 *
7687 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7688 * interrupts will be disabled.
7689 */
7690static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7691{
7692 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7693
7694#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7695 PGMRZDynMapFlushAutoSet(pVCpu);
7696#endif
7697
7698 /* Check force flag actions that might require us to go back to ring-3. */
7699 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7700 if (rc != VINF_SUCCESS)
7701 return rc;
7702
7703#ifndef IEM_VERIFICATION_MODE_FULL
7704 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7705 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7706 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7707 {
7708 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7709 RTGCPHYS GCPhysApicBase;
7710 GCPhysApicBase = pMixedCtx->msrApicBase;
7711 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7712
7713 /* Unalias any existing mapping. */
7714 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7715 AssertRCReturn(rc, rc);
7716
7717 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7718 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7719 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7720 AssertRCReturn(rc, rc);
7721
7722 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7723 }
7724#endif /* !IEM_VERIFICATION_MODE_FULL */
7725
7726 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7727 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7728
7729 /*
7730 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7731 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7732 */
7733 if (TRPMHasTrap(pVCpu))
7734 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7735 else if (!pVCpu->hm.s.Event.fPending)
7736 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7737
7738 /*
7739 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7740 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7741 */
7742 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7743 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7744 {
7745 Assert(rc == VINF_EM_RESET);
7746 return rc;
7747 }
7748
7749 /*
7750 * No longjmps to ring-3 from this point on!!!
7751 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7752 * This also disables flushing of the R0-logger instance (if any).
7753 */
7754 VMMRZCallRing3Disable(pVCpu);
7755
7756 /*
7757 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7758 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7759 *
7760 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7761 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7762 *
7763 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7764 * executing guest code.
7765 */
7766 pVmxTransient->uEflags = ASMIntDisableFlags();
7767 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7768 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7769 {
7770 hmR0VmxClearEventVmcs(pVCpu);
7771 ASMSetFlags(pVmxTransient->uEflags);
7772 VMMRZCallRing3Enable(pVCpu);
7773 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7774 return VINF_EM_RAW_TO_R3;
7775 }
7776 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7777 {
7778 hmR0VmxClearEventVmcs(pVCpu);
7779 ASMSetFlags(pVmxTransient->uEflags);
7780 VMMRZCallRing3Enable(pVCpu);
7781 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7782 return VINF_EM_RAW_INTERRUPT;
7783 }
7784
7785 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7786 pVCpu->hm.s.Event.fPending = false;
7787
7788 return VINF_SUCCESS;
7789}
7790
7791
7792/**
7793 * Prepares to run guest code in VT-x and we've committed to doing so. This
7794 * means there is no backing out to ring-3 or anywhere else at this
7795 * point.
7796 *
7797 * @param pVM Pointer to the VM.
7798 * @param pVCpu Pointer to the VMCPU.
7799 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7800 * out-of-sync. Make sure to update the required fields
7801 * before using them.
7802 * @param pVmxTransient Pointer to the VMX transient structure.
7803 *
7804 * @remarks Called with preemption disabled.
7805 * @remarks No-long-jump zone!!!
7806 */
7807static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7808{
7809 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7810 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7811 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7812
7813 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7814 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7815
7816 /*
7817 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7818 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7819 * Reload only the necessary state, the assertion will catch if other parts of the code
7820 * change.
7821 */
7822 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7823 {
7824 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7825 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7826 }
7827
7828#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7829 if (!CPUMIsGuestFPUStateActive(pVCpu))
7830 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7831 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7832#endif
7833
7834 if ( pVCpu->hm.s.fUseGuestFpu
7835 && !CPUMIsGuestFPUStateActive(pVCpu))
7836 {
7837 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7838 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
7839 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7840 }
7841
7842 /*
7843 * Load the host state bits as we may've been preempted (only happens when
7844 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7845 */
7846 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7847 {
7848 /* This ASSUMES that pfnStartVM has been set up already. */
7849 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7850 AssertRC(rc);
7851 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7852 }
7853 Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7854
7855 /*
7856 * Load the state shared between host and guest (FPU, debug).
7857 */
7858 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7859 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7860 AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7861
7862 /* Store status of the shared guest-host state at the time of VM-entry. */
7863#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7864 if (CPUMIsGuestInLongModeEx(pMixedCtx))
7865 {
7866 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7867 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7868 }
7869 else
7870#endif
7871 {
7872 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7873 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7874 }
7875 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7876
7877 /*
7878 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7879 */
7880 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7881 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7882
7883 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7884 RTCPUID idCurrentCpu = pCpu->idCpu;
7885 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7886 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7887 {
7888 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7889 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7890 }
7891
7892 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7893 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7894 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7895 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7896
7897 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7898
7899 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7900 to start executing. */
7901
7902#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7903 /*
7904 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7905 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7906 */
7907 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7908 {
7909 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7910 uint64_t u64HostTscAux = 0;
7911 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7912 AssertRC(rc2);
7913 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7914 }
7915#endif
7916}
7917
7918
7919/**
7920 * Performs some essential restoration of state after running guest code in
7921 * VT-x.
7922 *
7923 * @param pVM Pointer to the VM.
7924 * @param pVCpu Pointer to the VMCPU.
7925 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7926 * out-of-sync. Make sure to update the required fields
7927 * before using them.
7928 * @param pVmxTransient Pointer to the VMX transient structure.
7929 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7930 *
7931 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7932 *
7933 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7934 * unconditionally when it is safe to do so.
7935 */
7936static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7937{
7938 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7939
7940 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7941 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7942 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7943 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7944 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7945
7946 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7947 {
7948#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7949 /* Restore host's TSC_AUX. */
7950 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7951 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7952#endif
7953 /** @todo Find a way to fix hardcoding a guestimate. */
7954 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7955 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7956 }
7957
7958 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7959 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7960 Assert(!(ASMGetFlags() & X86_EFL_IF));
7961 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7962
7963#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7964 if (CPUMIsGuestFPUStateActive(pVCpu))
7965 {
7966 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7967 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7968 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7969 }
7970#endif
7971
7972 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7973 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7974 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7975 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7976
7977 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7978 uint32_t uExitReason;
7979 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7980 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7981 AssertRC(rc);
7982 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7983 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
7984
7985 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7986 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7987 {
7988 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7989 pVmxTransient->fVMEntryFailed));
7990 return;
7991 }
7992
7993 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7994 {
7995 /* Update the guest interruptibility-state from the VMCS. */
7996 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7997#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7998 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7999 AssertRC(rc);
8000#endif
8001 /*
8002 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8003 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8004 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8005 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8006 */
8007 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8008 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8009 {
8010 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8011 AssertRC(rc);
8012 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8013 }
8014 }
8015}
8016
8017
8018
8019/**
8020 * Runs the guest code using VT-x the normal way.
8021 *
8022 * @returns VBox status code.
8023 * @param pVM Pointer to the VM.
8024 * @param pVCpu Pointer to the VMCPU.
8025 * @param pCtx Pointer to the guest-CPU context.
8026 *
8027 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
8028 * @remarks Called with preemption disabled.
8029 */
8030static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8031{
8032 VMXTRANSIENT VmxTransient;
8033 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8034 int rc = VERR_INTERNAL_ERROR_5;
8035 uint32_t cLoops = 0;
8036
8037 for (;; cLoops++)
8038 {
8039 Assert(!HMR0SuspendPending());
8040 HMVMX_ASSERT_CPU_SAFE();
8041
8042 /* Preparatory work for running guest code, this may force us to return
8043 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8044 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8045 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8046 if (rc != VINF_SUCCESS)
8047 break;
8048
8049 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8050 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8051 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8052
8053 /* Restore any residual host-state and save any bits shared between host
8054 and guest into the guest-CPU state. Re-enables interrupts! */
8055 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8056
8057 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8058 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8059 {
8060 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8061 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8062 return rc;
8063 }
8064
8065 /* Handle the VM-exit. */
8066 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8067 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8068 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8069 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8070 HMVMX_START_EXIT_DISPATCH_PROF();
8071#ifdef HMVMX_USE_FUNCTION_TABLE
8072 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8073#else
8074 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8075#endif
8076 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8077 if (rc != VINF_SUCCESS)
8078 break;
8079 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8080 {
8081 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8082 rc = VINF_EM_RAW_INTERRUPT;
8083 break;
8084 }
8085 }
8086
8087 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8088 return rc;
8089}
8090
8091
8092/**
8093 * Single steps guest code using VT-x.
8094 *
8095 * @returns VBox status code.
8096 * @param pVM Pointer to the VM.
8097 * @param pVCpu Pointer to the VMCPU.
8098 * @param pCtx Pointer to the guest-CPU context.
8099 *
8100 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8101 * @remarks Called with preemption disabled.
8102 */
8103static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8104{
8105 VMXTRANSIENT VmxTransient;
8106 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8107 int rc = VERR_INTERNAL_ERROR_5;
8108 uint32_t cLoops = 0;
8109 uint16_t uCsStart = pCtx->cs.Sel;
8110 uint64_t uRipStart = pCtx->rip;
8111
8112 for (;; cLoops++)
8113 {
8114 Assert(!HMR0SuspendPending());
8115 HMVMX_ASSERT_CPU_SAFE();
8116
8117 /* Preparatory work for running guest code, this may force us to return
8118 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8119 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8120 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8121 if (rc != VINF_SUCCESS)
8122 break;
8123
8124 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8125 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8126 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8127
8128 /* Restore any residual host-state and save any bits shared between host
8129 and guest into the guest-CPU state. Re-enables interrupts! */
8130 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8131
8132 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8133 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8134 {
8135 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8136 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8137 return rc;
8138 }
8139
8140 /* Handle the VM-exit. */
8141 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8142 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8143 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8144 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8145 HMVMX_START_EXIT_DISPATCH_PROF();
8146#ifdef HMVMX_USE_FUNCTION_TABLE
8147 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8148#else
8149 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8150#endif
8151 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8152 if (rc != VINF_SUCCESS)
8153 break;
8154 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8155 {
8156 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8157 rc = VINF_EM_RAW_INTERRUPT;
8158 break;
8159 }
8160
8161 /*
8162 * Did the RIP change, if so, consider it a single step.
8163 * Otherwise, make sure one of the TFs gets set.
8164 */
8165 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8166 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8167 AssertRCReturn(rc2, rc2);
8168 if ( pCtx->rip != uRipStart
8169 || pCtx->cs.Sel != uCsStart)
8170 {
8171 rc = VINF_EM_DBG_STEPPED;
8172 break;
8173 }
8174 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8175 }
8176
8177 /*
8178 * Clear the X86_EFL_TF if necessary.
8179 */
8180 if (pVCpu->hm.s.fClearTrapFlag)
8181 {
8182 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8183 AssertRCReturn(rc2, rc2);
8184 pVCpu->hm.s.fClearTrapFlag = false;
8185 pCtx->eflags.Bits.u1TF = 0;
8186 }
8187 /** @todo there seems to be issues with the resume flag when the monitor trap
8188 * flag is pending without being used. Seen early in bios init when
8189 * accessing APIC page in prot mode. */
8190
8191 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8192 return rc;
8193}
8194
8195
8196/**
8197 * Runs the guest code using VT-x.
8198 *
8199 * @returns VBox status code.
8200 * @param pVM Pointer to the VM.
8201 * @param pVCpu Pointer to the VMCPU.
8202 * @param pCtx Pointer to the guest-CPU context.
8203 *
8204 * @remarks Called with preemption disabled.
8205 */
8206VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8207{
8208 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8209 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8210 HMVMX_ASSERT_PREEMPT_SAFE();
8211
8212 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8213
8214 int rc;
8215 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8216 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8217 else
8218 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8219
8220 if (rc == VERR_EM_INTERPRETER)
8221 rc = VINF_EM_RAW_EMULATE_INSTR;
8222 else if (rc == VINF_EM_RESET)
8223 rc = VINF_EM_TRIPLE_FAULT;
8224
8225 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8226 if (RT_FAILURE(rc2))
8227 {
8228 pVCpu->hm.s.u32HMError = rc;
8229 rc = rc2;
8230 }
8231 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8232 return rc;
8233}
8234
8235
8236#ifndef HMVMX_USE_FUNCTION_TABLE
8237DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8238{
8239 int rc;
8240 switch (rcReason)
8241 {
8242 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8243 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8244 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8245 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8246 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8247 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8248 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8249 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8250 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8251 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8252 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8253 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8254 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8255 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8256 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8257 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8258 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8259 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8260 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8261 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8262 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8263 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8264 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8265 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8266 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8267 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8268 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8269 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8270 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8271 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8272 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8273 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8274 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8275
8276 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8277 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8278 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8279 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8280 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8281 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8282 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8283 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8284 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8285
8286 case VMX_EXIT_VMCALL:
8287 case VMX_EXIT_VMCLEAR:
8288 case VMX_EXIT_VMLAUNCH:
8289 case VMX_EXIT_VMPTRLD:
8290 case VMX_EXIT_VMPTRST:
8291 case VMX_EXIT_VMREAD:
8292 case VMX_EXIT_VMRESUME:
8293 case VMX_EXIT_VMWRITE:
8294 case VMX_EXIT_VMXOFF:
8295 case VMX_EXIT_VMXON:
8296 case VMX_EXIT_INVEPT:
8297 case VMX_EXIT_INVVPID:
8298 case VMX_EXIT_VMFUNC:
8299 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8300 break;
8301 default:
8302 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8303 break;
8304 }
8305 return rc;
8306}
8307#endif
8308
8309#ifdef DEBUG
8310/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8311# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8312 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8313
8314# define HMVMX_ASSERT_PREEMPT_CPUID() \
8315 do \
8316 { \
8317 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8318 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8319 } while (0)
8320
8321# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8322 do { \
8323 AssertPtr(pVCpu); \
8324 AssertPtr(pMixedCtx); \
8325 AssertPtr(pVmxTransient); \
8326 Assert(pVmxTransient->fVMEntryFailed == false); \
8327 Assert(ASMIntAreEnabled()); \
8328 HMVMX_ASSERT_PREEMPT_SAFE(); \
8329 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8330 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8331 HMVMX_ASSERT_PREEMPT_SAFE(); \
8332 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8333 HMVMX_ASSERT_PREEMPT_CPUID(); \
8334 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8335 } while (0)
8336
8337# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8338 do { \
8339 Log4Func(("\n")); \
8340 } while(0)
8341#else /* Release builds */
8342# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8343# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8344#endif
8345
8346
8347/**
8348 * Advances the guest RIP after reading it from the VMCS.
8349 *
8350 * @returns VBox status code.
8351 * @param pVCpu Pointer to the VMCPU.
8352 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8353 * out-of-sync. Make sure to update the required fields
8354 * before using them.
8355 * @param pVmxTransient Pointer to the VMX transient structure.
8356 *
8357 * @remarks No-long-jump zone!!!
8358 */
8359DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8360{
8361 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8362 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8363 AssertRCReturn(rc, rc);
8364
8365 pMixedCtx->rip += pVmxTransient->cbInstr;
8366 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8367 return rc;
8368}
8369
8370
8371/**
8372 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8373 * and update error record fields accordingly.
8374 *
8375 * @return VMX_IGS_* return codes.
8376 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8377 * wrong with the guest state.
8378 *
8379 * @param pVM Pointer to the VM.
8380 * @param pVCpu Pointer to the VMCPU.
8381 * @param pCtx Pointer to the guest-CPU state.
8382 */
8383static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8384{
8385#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8386#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8387 uError = (err); \
8388 break; \
8389 } else do {} while (0)
8390/* Duplicate of IEM_IS_CANONICAL(). */
8391#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8392
8393 int rc;
8394 uint32_t uError = VMX_IGS_ERROR;
8395 uint32_t u32Val;
8396 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8397
8398 do
8399 {
8400 /*
8401 * CR0.
8402 */
8403 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8404 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8405 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8406 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8407 if (fUnrestrictedGuest)
8408 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8409
8410 uint32_t u32GuestCR0;
8411 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8412 AssertRCBreak(rc);
8413 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8414 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8415 if ( !fUnrestrictedGuest
8416 && (u32GuestCR0 & X86_CR0_PG)
8417 && !(u32GuestCR0 & X86_CR0_PE))
8418 {
8419 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8420 }
8421
8422 /*
8423 * CR4.
8424 */
8425 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8426 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8427
8428 uint32_t u32GuestCR4;
8429 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8430 AssertRCBreak(rc);
8431 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8432 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8433
8434 /*
8435 * IA32_DEBUGCTL MSR.
8436 */
8437 uint64_t u64Val;
8438 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8439 AssertRCBreak(rc);
8440 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8441 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8442 {
8443 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8444 }
8445 uint64_t u64DebugCtlMsr = u64Val;
8446
8447#ifdef VBOX_STRICT
8448 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8449 AssertRCBreak(rc);
8450 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8451#endif
8452 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8453
8454 /*
8455 * RIP and RFLAGS.
8456 */
8457 uint32_t u32Eflags;
8458#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8459 if (HMVMX_IS_64BIT_HOST_MODE())
8460 {
8461 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8462 AssertRCBreak(rc);
8463 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8464 if ( !fLongModeGuest
8465 || !pCtx->cs.Attr.n.u1Long)
8466 {
8467 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8468 }
8469 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8470 * must be identical if the "IA32e mode guest" VM-entry control is 1
8471 * and CS.L is 1. No check applies if the CPU supports 64
8472 * linear-address bits. */
8473
8474 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8475 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8476 AssertRCBreak(rc);
8477 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8478 VMX_IGS_RFLAGS_RESERVED);
8479 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8480 u32Eflags = u64Val;
8481 }
8482 else
8483#endif
8484 {
8485 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8486 AssertRCBreak(rc);
8487 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8488 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8489 }
8490
8491 if ( fLongModeGuest
8492 || ( fUnrestrictedGuest
8493 && !(u32GuestCR0 & X86_CR0_PE)))
8494 {
8495 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8496 }
8497
8498 uint32_t u32EntryInfo;
8499 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8500 AssertRCBreak(rc);
8501 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8502 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8503 {
8504 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8505 }
8506
8507 /*
8508 * 64-bit checks.
8509 */
8510#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8511 if (HMVMX_IS_64BIT_HOST_MODE())
8512 {
8513 if ( fLongModeGuest
8514 && !fUnrestrictedGuest)
8515 {
8516 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8517 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8518 }
8519
8520 if ( !fLongModeGuest
8521 && (u32GuestCR4 & X86_CR4_PCIDE))
8522 {
8523 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8524 }
8525
8526 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8527 * 51:32 beyond the processor's physical-address width are 0. */
8528
8529 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8530 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8531 {
8532 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8533 }
8534
8535 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8536 AssertRCBreak(rc);
8537 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8538
8539 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8540 AssertRCBreak(rc);
8541 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8542 }
8543#endif
8544
8545 /*
8546 * PERF_GLOBAL MSR.
8547 */
8548 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8549 {
8550 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8551 AssertRCBreak(rc);
8552 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8553 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8554 }
8555
8556 /*
8557 * PAT MSR.
8558 */
8559 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8560 {
8561 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8562 AssertRCBreak(rc);
8563 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8564 for (unsigned i = 0; i < 8; i++)
8565 {
8566 uint8_t u8Val = (u64Val & 0x7);
8567 if ( u8Val != 0 /* UC */
8568 || u8Val != 1 /* WC */
8569 || u8Val != 4 /* WT */
8570 || u8Val != 5 /* WP */
8571 || u8Val != 6 /* WB */
8572 || u8Val != 7 /* UC- */)
8573 {
8574 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8575 }
8576 u64Val >>= 3;
8577 }
8578 }
8579
8580 /*
8581 * EFER MSR.
8582 */
8583 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8584 {
8585 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8586 AssertRCBreak(rc);
8587 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8588 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8589 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8590 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8591 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8592 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8593 }
8594
8595 /*
8596 * Segment registers.
8597 */
8598 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8599 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8600 if (!(u32Eflags & X86_EFL_VM))
8601 {
8602 /* CS */
8603 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8604 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8605 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8606 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8607 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8608 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8609 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8610 /* CS cannot be loaded with NULL in protected mode. */
8611 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8612 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8613 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8614 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8615 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8616 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8617 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8618 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8619 else
8620 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8621
8622 /* SS */
8623 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8624 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8625 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8626 if ( !(pCtx->cr0 & X86_CR0_PE)
8627 || pCtx->cs.Attr.n.u4Type == 3)
8628 {
8629 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8630 }
8631 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8632 {
8633 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8634 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8635 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8636 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8637 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8638 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8639 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8640 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8641 }
8642
8643 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8644 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8645 {
8646 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8647 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8648 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8649 || pCtx->ds.Attr.n.u4Type > 11
8650 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8651 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8652 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8653 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8654 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8655 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8656 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8657 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8658 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8659 }
8660 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8661 {
8662 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8663 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8664 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8665 || pCtx->es.Attr.n.u4Type > 11
8666 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8667 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8668 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8669 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8670 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8671 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8672 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8673 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8674 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8675 }
8676 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8677 {
8678 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8679 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8680 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8681 || pCtx->fs.Attr.n.u4Type > 11
8682 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8683 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8684 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8685 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8686 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8687 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8688 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8689 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8690 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8691 }
8692 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8693 {
8694 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8695 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8696 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8697 || pCtx->gs.Attr.n.u4Type > 11
8698 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8699 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8700 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8701 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8702 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8703 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8704 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8705 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8706 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8707 }
8708 /* 64-bit capable CPUs. */
8709#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8710 if (HMVMX_IS_64BIT_HOST_MODE())
8711 {
8712 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8713 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8714 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8715 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8716 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8717 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8718 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8719 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8720 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8721 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8722 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8723 }
8724#endif
8725 }
8726 else
8727 {
8728 /* V86 mode checks. */
8729 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8730 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8731 {
8732 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8733 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8734 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8735 }
8736 else
8737 {
8738 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8739 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8740 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8741 }
8742
8743 /* CS */
8744 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8745 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8746 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8747 /* SS */
8748 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8749 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8750 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8751 /* DS */
8752 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8753 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8754 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8755 /* ES */
8756 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8757 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8758 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8759 /* FS */
8760 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8761 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8762 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8763 /* GS */
8764 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8765 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8766 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8767 /* 64-bit capable CPUs. */
8768#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8769 if (HMVMX_IS_64BIT_HOST_MODE())
8770 {
8771 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8772 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8773 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8774 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8775 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8776 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8777 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8778 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8779 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8780 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8781 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8782 }
8783#endif
8784 }
8785
8786 /*
8787 * TR.
8788 */
8789 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8790 /* 64-bit capable CPUs. */
8791#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8792 if (HMVMX_IS_64BIT_HOST_MODE())
8793 {
8794 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8795 }
8796#endif
8797 if (fLongModeGuest)
8798 {
8799 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8800 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8801 }
8802 else
8803 {
8804 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8805 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8806 VMX_IGS_TR_ATTR_TYPE_INVALID);
8807 }
8808 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8809 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8810 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8811 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8812 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8813 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8814 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8815 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8816
8817 /*
8818 * GDTR and IDTR.
8819 */
8820#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8821 if (HMVMX_IS_64BIT_HOST_MODE())
8822 {
8823 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8824 AssertRCBreak(rc);
8825 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8826
8827 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8828 AssertRCBreak(rc);
8829 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8830 }
8831#endif
8832
8833 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8834 AssertRCBreak(rc);
8835 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8836
8837 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8838 AssertRCBreak(rc);
8839 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8840
8841 /*
8842 * Guest Non-Register State.
8843 */
8844 /* Activity State. */
8845 uint32_t u32ActivityState;
8846 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8847 AssertRCBreak(rc);
8848 HMVMX_CHECK_BREAK( !u32ActivityState
8849 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8850 VMX_IGS_ACTIVITY_STATE_INVALID);
8851 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8852 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8853 uint32_t u32IntrState;
8854 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8855 AssertRCBreak(rc);
8856 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8857 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8858 {
8859 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8860 }
8861
8862 /** @todo Activity state and injecting interrupts. Left as a todo since we
8863 * currently don't use activity states but ACTIVE. */
8864
8865 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8866 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8867
8868 /* Guest interruptibility-state. */
8869 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8870 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8871 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8872 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8873 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8874 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8875 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8876 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8877 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8878 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
8879 {
8880 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8881 {
8882 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8883 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8884 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8885 }
8886 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8887 {
8888 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8889 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8890 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8891 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8892 }
8893 }
8894 /** @todo Assumes the processor is not in SMM. */
8895 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8896 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8897 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8898 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8899 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8900 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8901 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8902 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8903 {
8904 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8905 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8906 }
8907
8908 /* Pending debug exceptions. */
8909 if (HMVMX_IS_64BIT_HOST_MODE())
8910 {
8911 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8912 AssertRCBreak(rc);
8913 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8914 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8915 u32Val = u64Val; /* For pending debug exceptions checks below. */
8916 }
8917 else
8918 {
8919 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8920 AssertRCBreak(rc);
8921 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8922 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8923 }
8924
8925 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8926 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8927 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8928 {
8929 if ( (u32Eflags & X86_EFL_TF)
8930 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8931 {
8932 /* Bit 14 is PendingDebug.BS. */
8933 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8934 }
8935 if ( !(u32Eflags & X86_EFL_TF)
8936 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8937 {
8938 /* Bit 14 is PendingDebug.BS. */
8939 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8940 }
8941 }
8942
8943 /* VMCS link pointer. */
8944 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8945 AssertRCBreak(rc);
8946 if (u64Val != UINT64_C(0xffffffffffffffff))
8947 {
8948 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8949 /** @todo Bits beyond the processor's physical-address width MBZ. */
8950 /** @todo 32-bit located in memory referenced by value of this field (as a
8951 * physical address) must contain the processor's VMCS revision ID. */
8952 /** @todo SMM checks. */
8953 }
8954
8955 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8956
8957 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8958 if (uError == VMX_IGS_ERROR)
8959 uError = VMX_IGS_REASON_NOT_FOUND;
8960 } while (0);
8961
8962 pVCpu->hm.s.u32HMError = uError;
8963 return uError;
8964
8965#undef HMVMX_ERROR_BREAK
8966#undef HMVMX_CHECK_BREAK
8967#undef HMVMX_IS_CANONICAL
8968}
8969
8970/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8971/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8972/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8973
8974/** @name VM-exit handlers.
8975 * @{
8976 */
8977
8978/**
8979 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8980 */
8981HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8982{
8983 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8984 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8985 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8986#if HC_ARCH_BITS == 64
8987 Assert(ASMIntAreEnabled());
8988 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8989 return VINF_SUCCESS;
8990#endif
8991 return VINF_EM_RAW_INTERRUPT;
8992}
8993
8994
8995/**
8996 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8997 */
8998HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8999{
9000 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9001 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9002
9003 int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
9004 AssertRCReturn(rc, rc);
9005
9006 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9007 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9008 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9009 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9010
9011 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9012 {
9013 /*
9014 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9015 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9016 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9017 *
9018 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9019 */
9020 VMXDispatchHostNmi();
9021 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9022 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9023 return VINF_SUCCESS;
9024 }
9025
9026 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9027 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9028 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9029 {
9030 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9031 return VINF_SUCCESS;
9032 }
9033 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9034 {
9035 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9036 return rc;
9037 }
9038
9039 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9040 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9041 switch (uIntType)
9042 {
9043 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9044 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
9045 /* no break */
9046 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9047 {
9048 switch (uVector)
9049 {
9050 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9051 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9052 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9053 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9054 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9055 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9056#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9057 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9058 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9059 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9060 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9061 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9062 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9063 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9064 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9065 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9066 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9067#endif
9068 default:
9069 {
9070 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9071 AssertRCReturn(rc, rc);
9072
9073 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9074 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9075 {
9076 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9077 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9078 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9079
9080 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9081 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
9082 AssertRCReturn(rc, rc);
9083 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9084 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9085 0 /* GCPtrFaultAddress */);
9086 AssertRCReturn(rc, rc);
9087 }
9088 else
9089 {
9090 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9091 pVCpu->hm.s.u32HMError = uVector;
9092 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9093 }
9094 break;
9095 }
9096 }
9097 break;
9098 }
9099
9100 default:
9101 {
9102 pVCpu->hm.s.u32HMError = uExitIntInfo;
9103 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9104 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9105 break;
9106 }
9107 }
9108 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9109 return rc;
9110}
9111
9112
9113/**
9114 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9115 */
9116HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9117{
9118 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9119
9120 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9121 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9122 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9123 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9124 AssertRCReturn(rc, rc);
9125
9126 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9127 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9128 return VINF_SUCCESS;
9129}
9130
9131
9132/**
9133 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9134 */
9135HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9136{
9137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9138 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9139 HMVMX_RETURN_UNEXPECTED_EXIT();
9140}
9141
9142
9143/**
9144 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9145 */
9146HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9147{
9148 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9150 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9151}
9152
9153
9154/**
9155 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9156 */
9157HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9158{
9159 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9161 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9162}
9163
9164
9165/**
9166 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9167 */
9168HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9169{
9170 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9171 PVM pVM = pVCpu->CTX_SUFF(pVM);
9172 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9173 if (RT_LIKELY(rc == VINF_SUCCESS))
9174 {
9175 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9176 Assert(pVmxTransient->cbInstr == 2);
9177 }
9178 else
9179 {
9180 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9181 rc = VERR_EM_INTERPRETER;
9182 }
9183 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9184 return rc;
9185}
9186
9187
9188/**
9189 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9190 */
9191HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9192{
9193 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9194 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9195 AssertRCReturn(rc, rc);
9196
9197 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9198 return VINF_EM_RAW_EMULATE_INSTR;
9199
9200 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9201 HMVMX_RETURN_UNEXPECTED_EXIT();
9202}
9203
9204
9205/**
9206 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9207 */
9208HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9209{
9210 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9211 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9212 AssertRCReturn(rc, rc);
9213
9214 PVM pVM = pVCpu->CTX_SUFF(pVM);
9215 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9216 if (RT_LIKELY(rc == VINF_SUCCESS))
9217 {
9218 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9219 Assert(pVmxTransient->cbInstr == 2);
9220 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9221 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9222 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9223 }
9224 else
9225 {
9226 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9227 rc = VERR_EM_INTERPRETER;
9228 }
9229 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9230 return rc;
9231}
9232
9233
9234/**
9235 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9236 */
9237HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9238{
9239 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9240 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9241 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9242 AssertRCReturn(rc, rc);
9243
9244 PVM pVM = pVCpu->CTX_SUFF(pVM);
9245 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9246 if (RT_LIKELY(rc == VINF_SUCCESS))
9247 {
9248 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9249 Assert(pVmxTransient->cbInstr == 3);
9250 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9251 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9252 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9253 }
9254 else
9255 {
9256 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9257 rc = VERR_EM_INTERPRETER;
9258 }
9259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9260 return rc;
9261}
9262
9263
9264/**
9265 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9266 */
9267HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9268{
9269 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9270 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9271 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9272 AssertRCReturn(rc, rc);
9273
9274 PVM pVM = pVCpu->CTX_SUFF(pVM);
9275 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9276 if (RT_LIKELY(rc == VINF_SUCCESS))
9277 {
9278 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9279 Assert(pVmxTransient->cbInstr == 2);
9280 }
9281 else
9282 {
9283 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9284 rc = VERR_EM_INTERPRETER;
9285 }
9286 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9287 return rc;
9288}
9289
9290
9291/**
9292 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9293 */
9294HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9295{
9296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9297 PVM pVM = pVCpu->CTX_SUFF(pVM);
9298 Assert(!pVM->hm.s.fNestedPaging);
9299
9300 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9301 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9302 AssertRCReturn(rc, rc);
9303
9304 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9305 rc = VBOXSTRICTRC_VAL(rc2);
9306 if (RT_LIKELY(rc == VINF_SUCCESS))
9307 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9308 else
9309 {
9310 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9311 pVmxTransient->uExitQualification, rc));
9312 }
9313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9314 return rc;
9315}
9316
9317
9318/**
9319 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9320 */
9321HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9322{
9323 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9324 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9325 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9326 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9327 AssertRCReturn(rc, rc);
9328
9329 PVM pVM = pVCpu->CTX_SUFF(pVM);
9330 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9331 if (RT_LIKELY(rc == VINF_SUCCESS))
9332 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9333 else
9334 {
9335 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9336 rc = VERR_EM_INTERPRETER;
9337 }
9338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9339 return rc;
9340}
9341
9342
9343/**
9344 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9345 */
9346HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9347{
9348 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9349 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9350 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9351 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9352 AssertRCReturn(rc, rc);
9353
9354 PVM pVM = pVCpu->CTX_SUFF(pVM);
9355 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9356 rc = VBOXSTRICTRC_VAL(rc2);
9357 if (RT_LIKELY( rc == VINF_SUCCESS
9358 || rc == VINF_EM_HALT))
9359 {
9360 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9361 AssertRCReturn(rc3, rc3);
9362
9363 if ( rc == VINF_EM_HALT
9364 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9365 {
9366 rc = VINF_SUCCESS;
9367 }
9368 }
9369 else
9370 {
9371 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9372 rc = VERR_EM_INTERPRETER;
9373 }
9374 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9375 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9377 return rc;
9378}
9379
9380
9381/**
9382 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9383 */
9384HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9385{
9386 /*
9387 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9388 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9389 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9390 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9391 */
9392 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9393 HMVMX_RETURN_UNEXPECTED_EXIT();
9394}
9395
9396
9397/**
9398 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9399 */
9400HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9401{
9402 /*
9403 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9404 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9405 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9406 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9407 */
9408 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9409 HMVMX_RETURN_UNEXPECTED_EXIT();
9410}
9411
9412
9413/**
9414 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9415 */
9416HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9417{
9418 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9419 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9420 HMVMX_RETURN_UNEXPECTED_EXIT();
9421}
9422
9423
9424/**
9425 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9426 */
9427HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9428{
9429 /*
9430 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9431 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9432 * See Intel spec. 25.3 "Other Causes of VM-exits".
9433 */
9434 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9435 HMVMX_RETURN_UNEXPECTED_EXIT();
9436}
9437
9438
9439/**
9440 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9441 * VM-exit.
9442 */
9443HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9444{
9445 /*
9446 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9447 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9448 *
9449 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9450 * See Intel spec. "23.8 Restrictions on VMX operation".
9451 */
9452 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9453 return VINF_SUCCESS;
9454}
9455
9456
9457/**
9458 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9459 * VM-exit.
9460 */
9461HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9462{
9463 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9464 return VINF_EM_RESET;
9465}
9466
9467
9468/**
9469 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9470 */
9471HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9472{
9473 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9474 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9475 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9476 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9477 AssertRCReturn(rc, rc);
9478
9479 pMixedCtx->rip++;
9480 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9481 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9482 rc = VINF_SUCCESS;
9483 else
9484 rc = VINF_EM_HALT;
9485
9486 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9487 return rc;
9488}
9489
9490
9491/**
9492 * VM-exit handler for instructions that result in a #UD exception delivered to
9493 * the guest.
9494 */
9495HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9496{
9497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9498 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9499 return VINF_SUCCESS;
9500}
9501
9502
9503/**
9504 * VM-exit handler for expiry of the VMX preemption timer.
9505 */
9506HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9507{
9508 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9509
9510 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9511 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9512
9513 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9514 PVM pVM = pVCpu->CTX_SUFF(pVM);
9515 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9516 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9517 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9518}
9519
9520
9521/**
9522 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9523 */
9524HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9525{
9526 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9527
9528 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9529 /** @todo check if XSETBV is supported by the recompiler. */
9530 return VERR_EM_INTERPRETER;
9531}
9532
9533
9534/**
9535 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9536 */
9537HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9538{
9539 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9540
9541 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9542 /** @todo implement EMInterpretInvpcid() */
9543 return VERR_EM_INTERPRETER;
9544}
9545
9546
9547/**
9548 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9549 * Error VM-exit.
9550 */
9551HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9552{
9553 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9554 AssertRCReturn(rc, rc);
9555
9556 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9557 NOREF(uInvalidReason);
9558
9559#ifdef VBOX_STRICT
9560 uint32_t uIntrState;
9561 HMVMXHCUINTREG uHCReg;
9562 uint64_t u64Val;
9563 uint32_t u32Val;
9564
9565 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9566 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9567 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9568 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9569 AssertRCReturn(rc, rc);
9570
9571 Log4(("uInvalidReason %u\n", uInvalidReason));
9572 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9573 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9574 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9575 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9576
9577 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9578 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9579 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9580 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9581 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9582 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9583 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9584 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9585 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9586 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9587 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9588 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9589#endif
9590
9591 PVM pVM = pVCpu->CTX_SUFF(pVM);
9592 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9593
9594 return VERR_VMX_INVALID_GUEST_STATE;
9595}
9596
9597
9598/**
9599 * VM-exit handler for VM-entry failure due to an MSR-load
9600 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9601 */
9602HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9603{
9604 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9605 HMVMX_RETURN_UNEXPECTED_EXIT();
9606}
9607
9608
9609/**
9610 * VM-exit handler for VM-entry failure due to a machine-check event
9611 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9612 */
9613HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9614{
9615 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9616 HMVMX_RETURN_UNEXPECTED_EXIT();
9617}
9618
9619
9620/**
9621 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9622 * theory.
9623 */
9624HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9625{
9626 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9627 return VERR_VMX_UNDEFINED_EXIT_CODE;
9628}
9629
9630
9631/**
9632 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9633 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9634 * Conditional VM-exit.
9635 */
9636HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9637{
9638 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9639
9640 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9641 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9642 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9643 return VERR_EM_INTERPRETER;
9644 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9645 HMVMX_RETURN_UNEXPECTED_EXIT();
9646}
9647
9648
9649/**
9650 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9651 */
9652HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9653{
9654 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9655
9656 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9657 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9658 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9659 return VERR_EM_INTERPRETER;
9660 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9661 HMVMX_RETURN_UNEXPECTED_EXIT();
9662}
9663
9664
9665/**
9666 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9667 */
9668HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9669{
9670 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9671
9672 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9673 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9674 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9675 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9676 AssertRCReturn(rc, rc);
9677 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9678
9679 PVM pVM = pVCpu->CTX_SUFF(pVM);
9680 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9681 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9682 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9683 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9684
9685 if (RT_LIKELY(rc == VINF_SUCCESS))
9686 {
9687 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9688 Assert(pVmxTransient->cbInstr == 2);
9689 }
9690 return rc;
9691}
9692
9693
9694/**
9695 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9696 */
9697HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9698{
9699 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9700 PVM pVM = pVCpu->CTX_SUFF(pVM);
9701 int rc = VINF_SUCCESS;
9702
9703 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9704 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9705 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9706 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9707 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9708 AssertRCReturn(rc, rc);
9709 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9710
9711 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9712 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9713 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9714
9715 if (RT_LIKELY(rc == VINF_SUCCESS))
9716 {
9717 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9718
9719 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9720 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9721 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9722 {
9723 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9724 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9725 EMInterpretWrmsr() changes it. */
9726 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9727 }
9728 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9729 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9730 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9731 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9732
9733 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9734 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9735 {
9736 switch (pMixedCtx->ecx)
9737 {
9738 case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9739 case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9740 case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9741 case MSR_K8_FS_BASE: /* no break */
9742 case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9743 case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9744 }
9745 }
9746#ifdef VBOX_STRICT
9747 else
9748 {
9749 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9750 switch (pMixedCtx->ecx)
9751 {
9752 case MSR_IA32_SYSENTER_CS:
9753 case MSR_IA32_SYSENTER_EIP:
9754 case MSR_IA32_SYSENTER_ESP:
9755 case MSR_K8_FS_BASE:
9756 case MSR_K8_GS_BASE:
9757 {
9758 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9759 HMVMX_RETURN_UNEXPECTED_EXIT();
9760 }
9761
9762 case MSR_K8_LSTAR:
9763 case MSR_K6_STAR:
9764 case MSR_K8_SF_MASK:
9765 case MSR_K8_TSC_AUX:
9766 case MSR_K8_KERNEL_GS_BASE:
9767 {
9768 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9769 pMixedCtx->ecx));
9770 HMVMX_RETURN_UNEXPECTED_EXIT();
9771 }
9772 }
9773 }
9774#endif /* VBOX_STRICT */
9775 }
9776 return rc;
9777}
9778
9779
9780/**
9781 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9782 */
9783HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9784{
9785 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9786
9787 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9788 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9789 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9790 return VERR_EM_INTERPRETER;
9791 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9792 HMVMX_RETURN_UNEXPECTED_EXIT();
9793}
9794
9795
9796/**
9797 * VM-exit handler for when the TPR value is lowered below the specified
9798 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9799 */
9800HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9801{
9802 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9803 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9804
9805 /*
9806 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9807 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9808 * resume guest execution.
9809 */
9810 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9811 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9812 return VINF_SUCCESS;
9813}
9814
9815
9816/**
9817 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9818 * VM-exit.
9819 *
9820 * @retval VINF_SUCCESS when guest execution can continue.
9821 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9822 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9823 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9824 * recompiler.
9825 */
9826HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9827{
9828 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9829 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9830 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9831 AssertRCReturn(rc, rc);
9832
9833 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9834 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9835 PVM pVM = pVCpu->CTX_SUFF(pVM);
9836 switch (uAccessType)
9837 {
9838 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9839 {
9840#if 0
9841 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9842 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9843#else
9844 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9845 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9846 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9847#endif
9848 AssertRCReturn(rc, rc);
9849
9850 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9851 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9852 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9853 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9854
9855 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9856 {
9857 case 0: /* CR0 */
9858 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9859 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9860 break;
9861 case 2: /* CR2 */
9862 /* Nothing to do here, CR2 it's not part of the VMCS. */
9863 break;
9864 case 3: /* CR3 */
9865 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9866 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9867 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9868 break;
9869 case 4: /* CR4 */
9870 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9871 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9872 break;
9873 case 8: /* CR8 */
9874 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9875 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9876 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9877 break;
9878 default:
9879 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9880 break;
9881 }
9882
9883 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9884 break;
9885 }
9886
9887 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9888 {
9889 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9890 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9891 AssertRCReturn(rc, rc);
9892 Assert( !pVM->hm.s.fNestedPaging
9893 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9894 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9895
9896 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9897 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9898 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9899
9900 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9901 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9902 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9903 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9904 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9905 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9906 break;
9907 }
9908
9909 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9910 {
9911 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9912 AssertRCReturn(rc, rc);
9913 rc = EMInterpretCLTS(pVM, pVCpu);
9914 AssertRCReturn(rc, rc);
9915 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9917 Log4(("CRX CLTS write rc=%d\n", rc));
9918 break;
9919 }
9920
9921 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9922 {
9923 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9924 AssertRCReturn(rc, rc);
9925 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9926 if (RT_LIKELY(rc == VINF_SUCCESS))
9927 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9928 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9929 Log4(("CRX LMSW write rc=%d\n", rc));
9930 break;
9931 }
9932
9933 default:
9934 {
9935 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9936 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9937 }
9938 }
9939
9940 /* Validate possible error codes. */
9941 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9942 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9943 if (RT_SUCCESS(rc))
9944 {
9945 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9946 AssertRCReturn(rc2, rc2);
9947 }
9948
9949 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9950 return rc;
9951}
9952
9953
9954/**
9955 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9956 * VM-exit.
9957 */
9958HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9959{
9960 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9961 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9962
9963 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9964 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9965 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9966 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9967 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9968 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9969 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9970 AssertRCReturn(rc2, rc2);
9971
9972 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9973 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9974 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9975 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9976 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9977 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9978 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9979
9980 /* I/O operation lookup arrays. */
9981 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9982 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9983
9984 VBOXSTRICTRC rcStrict;
9985 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9986 const uint32_t cbInstr = pVmxTransient->cbInstr;
9987 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9988 PVM pVM = pVCpu->CTX_SUFF(pVM);
9989 if (fIOString)
9990 {
9991#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9992 /*
9993 * INS/OUTS - I/O String instruction.
9994 *
9995 * Use instruction-information if available, otherwise fall back on
9996 * interpreting the instruction.
9997 */
9998 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9999 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10000 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10001 {
10002 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
10003 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10004 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10005 AssertRCReturn(rc2, rc2);
10006 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10007 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10008 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10009 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10010 if (fIOWrite)
10011 {
10012 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10013 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10014 }
10015 else
10016 {
10017 /*
10018 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10019 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10020 * See Intel Instruction spec. for "INS".
10021 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10022 */
10023 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10024 }
10025 }
10026 else
10027 {
10028 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10029 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10030 AssertRCReturn(rc2, rc2);
10031 rcStrict = IEMExecOne(pVCpu);
10032 }
10033 /** @todo IEM needs to be setting these flags somehow. */
10034 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10035 fUpdateRipAlready = true;
10036#else
10037 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10038 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
10039 if (RT_SUCCESS(rcStrict))
10040 {
10041 if (fIOWrite)
10042 {
10043 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10044 (DISCPUMODE)pDis->uAddrMode, cbValue);
10045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10046 }
10047 else
10048 {
10049 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10050 (DISCPUMODE)pDis->uAddrMode, cbValue);
10051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10052 }
10053 }
10054 else
10055 {
10056 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10057 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10058 }
10059#endif
10060 }
10061 else
10062 {
10063 /*
10064 * IN/OUT - I/O instruction.
10065 */
10066 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10067 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10068 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10069 if (fIOWrite)
10070 {
10071 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10072 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10073 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10075 }
10076 else
10077 {
10078 uint32_t u32Result = 0;
10079 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10080 if (IOM_SUCCESS(rcStrict))
10081 {
10082 /* Save result of I/O IN instr. in AL/AX/EAX. */
10083 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10084 }
10085 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10086 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10087 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10088 }
10089 }
10090
10091 if (IOM_SUCCESS(rcStrict))
10092 {
10093 if (!fUpdateRipAlready)
10094 {
10095 pMixedCtx->rip += cbInstr;
10096 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10097 }
10098
10099 /* INS & OUTS with REP prefix modify RFLAGS. */
10100 if (fIOString)
10101 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10102
10103 /*
10104 * If any I/O breakpoints are armed, we need to check if one triggered
10105 * and take appropriate action.
10106 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10107 */
10108 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10109 AssertRCReturn(rc2, rc2);
10110
10111 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10112 * execution engines about whether hyper BPs and such are pending. */
10113 uint32_t const uDr7 = pMixedCtx->dr[7];
10114 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10115 && X86_DR7_ANY_RW_IO(uDr7)
10116 && (pMixedCtx->cr4 & X86_CR4_DE))
10117 || DBGFBpIsHwIoArmed(pVM)))
10118 {
10119 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10120
10121 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10122 VMMRZCallRing3Disable(pVCpu);
10123 HM_DISABLE_PREEMPT_IF_NEEDED();
10124
10125 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10126
10127 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10128 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10129 {
10130 /* Raise #DB. */
10131 if (fIsGuestDbgActive)
10132 ASMSetDR6(pMixedCtx->dr[6]);
10133 if (pMixedCtx->dr[7] != uDr7)
10134 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10135
10136 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10137 }
10138 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10139 else if ( rcStrict2 != VINF_SUCCESS
10140 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10141 rcStrict = rcStrict2;
10142
10143 HM_RESTORE_PREEMPT_IF_NEEDED();
10144 VMMRZCallRing3Enable(pVCpu);
10145 }
10146 }
10147
10148#ifdef DEBUG
10149 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10150 Assert(!fIOWrite);
10151 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10152 Assert(fIOWrite);
10153 else
10154 {
10155 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10156 * statuses, that the VMM device and some others may return. See
10157 * IOM_SUCCESS() for guidance. */
10158 AssertMsg( RT_FAILURE(rcStrict)
10159 || rcStrict == VINF_SUCCESS
10160 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10161 || rcStrict == VINF_EM_DBG_BREAKPOINT
10162 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10163 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10164 }
10165#endif
10166
10167 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10168 return VBOXSTRICTRC_TODO(rcStrict);
10169}
10170
10171
10172/**
10173 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10174 * VM-exit.
10175 */
10176HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10177{
10178 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10179
10180 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10181 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10182 AssertRCReturn(rc, rc);
10183 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10184 {
10185 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10186 AssertRCReturn(rc, rc);
10187 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10188 {
10189 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10190
10191 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10192 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10193 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10194 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10195 {
10196 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10197 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10198
10199 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10200 Assert(!pVCpu->hm.s.Event.fPending);
10201 pVCpu->hm.s.Event.fPending = true;
10202 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10203 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10204 AssertRCReturn(rc, rc);
10205 if (fErrorCodeValid)
10206 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10207 else
10208 pVCpu->hm.s.Event.u32ErrCode = 0;
10209 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10210 && uVector == X86_XCPT_PF)
10211 {
10212 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10213 }
10214
10215 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10216 }
10217 }
10218 }
10219
10220 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10221 * emulation. */
10222 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10223 return VERR_EM_INTERPRETER;
10224}
10225
10226
10227/**
10228 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10229 */
10230HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10231{
10232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10233 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10234 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10235 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10236 AssertRCReturn(rc, rc);
10237 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10238 return VINF_EM_DBG_STEPPED;
10239}
10240
10241
10242/**
10243 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10244 */
10245HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10246{
10247 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10248
10249 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10250 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10251 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10252 return VINF_SUCCESS;
10253 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10254 return rc;
10255
10256#if 0
10257 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10258 * just sync the whole thing. */
10259 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10260#else
10261 /* Aggressive state sync. for now. */
10262 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10263 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10264 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10265#endif
10266 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10267 AssertRCReturn(rc, rc);
10268
10269 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10270 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10271 switch (uAccessType)
10272 {
10273 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10274 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10275 {
10276 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10277 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10278 {
10279 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10280 }
10281
10282 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10283 GCPhys &= PAGE_BASE_GC_MASK;
10284 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10285 PVM pVM = pVCpu->CTX_SUFF(pVM);
10286 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10287 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10288
10289 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10290 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10291 CPUMCTX2CORE(pMixedCtx), GCPhys);
10292 rc = VBOXSTRICTRC_VAL(rc2);
10293 Log4(("ApicAccess rc=%d\n", rc));
10294 if ( rc == VINF_SUCCESS
10295 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10296 || rc == VERR_PAGE_NOT_PRESENT)
10297 {
10298 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10299 | HM_CHANGED_GUEST_RSP
10300 | HM_CHANGED_GUEST_RFLAGS
10301 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10302 rc = VINF_SUCCESS;
10303 }
10304 break;
10305 }
10306
10307 default:
10308 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10309 rc = VINF_EM_RAW_EMULATE_INSTR;
10310 break;
10311 }
10312
10313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10314 return rc;
10315}
10316
10317
10318/**
10319 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10320 * VM-exit.
10321 */
10322HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10323{
10324 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10325
10326 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10327 if (pVmxTransient->fWasGuestDebugStateActive)
10328 {
10329 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10330 HMVMX_RETURN_UNEXPECTED_EXIT();
10331 }
10332
10333 int rc = VERR_INTERNAL_ERROR_5;
10334 if ( !DBGFIsStepping(pVCpu)
10335 && !pVCpu->hm.s.fSingleInstruction
10336 && !pVmxTransient->fWasHyperDebugStateActive)
10337 {
10338 /* Don't intercept MOV DRx and #DB any more. */
10339 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10340 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10341 AssertRCReturn(rc, rc);
10342
10343 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10344 {
10345#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10346 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10347 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10348 AssertRCReturn(rc, rc);
10349#endif
10350 }
10351
10352 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10353 VMMRZCallRing3Disable(pVCpu);
10354 HM_DISABLE_PREEMPT_IF_NEEDED();
10355
10356 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10357 PVM pVM = pVCpu->CTX_SUFF(pVM);
10358 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10359 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10360
10361 HM_RESTORE_PREEMPT_IF_NEEDED();
10362 VMMRZCallRing3Enable(pVCpu);
10363
10364#ifdef VBOX_WITH_STATISTICS
10365 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10366 AssertRCReturn(rc, rc);
10367 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10369 else
10370 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10371#endif
10372 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10373 return VINF_SUCCESS;
10374 }
10375
10376 /*
10377 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10378 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10379 */
10380 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10381 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10382 AssertRCReturn(rc, rc);
10383 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10384
10385 PVM pVM = pVCpu->CTX_SUFF(pVM);
10386 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10387 {
10388 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10389 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10390 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10391 if (RT_SUCCESS(rc))
10392 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10394 }
10395 else
10396 {
10397 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10398 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10399 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10400 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10401 }
10402
10403 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10404 if (RT_SUCCESS(rc))
10405 {
10406 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10407 AssertRCReturn(rc2, rc2);
10408 }
10409 return rc;
10410}
10411
10412
10413/**
10414 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10415 * Conditional VM-exit.
10416 */
10417HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10418{
10419 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10420 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10421
10422 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10423 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10424 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10425 return VINF_SUCCESS;
10426 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10427 return rc;
10428
10429 RTGCPHYS GCPhys = 0;
10430 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10431
10432#if 0
10433 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10434#else
10435 /* Aggressive state sync. for now. */
10436 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10437 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10438 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10439#endif
10440 AssertRCReturn(rc, rc);
10441
10442 /*
10443 * If we succeed, resume guest execution.
10444 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10445 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10446 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10447 * weird case. See @bugref{6043}.
10448 */
10449 PVM pVM = pVCpu->CTX_SUFF(pVM);
10450 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10451 rc = VBOXSTRICTRC_VAL(rc2);
10452 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10453 if ( rc == VINF_SUCCESS
10454 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10455 || rc == VERR_PAGE_NOT_PRESENT)
10456 {
10457 /* Successfully handled MMIO operation. */
10458 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10459 | HM_CHANGED_GUEST_RSP
10460 | HM_CHANGED_GUEST_RFLAGS
10461 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10462 rc = VINF_SUCCESS;
10463 }
10464 return rc;
10465}
10466
10467
10468/**
10469 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10470 * VM-exit.
10471 */
10472HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10473{
10474 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10475 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10476
10477 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10478 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10479 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10480 return VINF_SUCCESS;
10481 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10482 return rc;
10483
10484 RTGCPHYS GCPhys = 0;
10485 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10486 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10487#if 0
10488 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10489#else
10490 /* Aggressive state sync. for now. */
10491 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10492 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10493 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10494#endif
10495 AssertRCReturn(rc, rc);
10496
10497 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10498 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10499
10500 RTGCUINT uErrorCode = 0;
10501 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10502 uErrorCode |= X86_TRAP_PF_ID;
10503 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10504 uErrorCode |= X86_TRAP_PF_RW;
10505 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10506 uErrorCode |= X86_TRAP_PF_P;
10507
10508 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10509
10510 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10511 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10512
10513 /* Handle the pagefault trap for the nested shadow table. */
10514 PVM pVM = pVCpu->CTX_SUFF(pVM);
10515 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10516 TRPMResetTrap(pVCpu);
10517
10518 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10519 if ( rc == VINF_SUCCESS
10520 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10521 || rc == VERR_PAGE_NOT_PRESENT)
10522 {
10523 /* Successfully synced our nested page tables. */
10524 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10525 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10526 | HM_CHANGED_GUEST_RSP
10527 | HM_CHANGED_GUEST_RFLAGS);
10528 return VINF_SUCCESS;
10529 }
10530
10531 Log4(("EPT return to ring-3 rc=%d\n"));
10532 return rc;
10533}
10534
10535/** @} */
10536
10537/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10538/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10539/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10540
10541/** @name VM-exit exception handlers.
10542 * @{
10543 */
10544
10545/**
10546 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10547 */
10548static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10549{
10550 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10551 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10552
10553 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10554 AssertRCReturn(rc, rc);
10555
10556 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10557 {
10558 /* Old-style FPU error reporting needs some extra work. */
10559 /** @todo don't fall back to the recompiler, but do it manually. */
10560 return VERR_EM_INTERPRETER;
10561 }
10562
10563 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10564 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10565 return rc;
10566}
10567
10568
10569/**
10570 * VM-exit exception handler for #BP (Breakpoint exception).
10571 */
10572static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10573{
10574 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10575 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10576
10577 /** @todo Try optimize this by not saving the entire guest state unless
10578 * really needed. */
10579 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10580 AssertRCReturn(rc, rc);
10581
10582 PVM pVM = pVCpu->CTX_SUFF(pVM);
10583 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10584 if (rc == VINF_EM_RAW_GUEST_TRAP)
10585 {
10586 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10587 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10588 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10589 AssertRCReturn(rc, rc);
10590
10591 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10592 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10593 }
10594
10595 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10596 return rc;
10597}
10598
10599
10600/**
10601 * VM-exit exception handler for #DB (Debug exception).
10602 */
10603static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10604{
10605 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10606 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10607 Log6(("XcptDB\n"));
10608
10609 /*
10610 * Get the DR6-like values from the exit qualification and pass it to DBGF
10611 * for processing.
10612 */
10613 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10614 AssertRCReturn(rc, rc);
10615
10616 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10617 uint64_t uDR6 = X86_DR6_INIT_VAL;
10618 uDR6 |= ( pVmxTransient->uExitQualification
10619 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10620
10621 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10622 if (rc == VINF_EM_RAW_GUEST_TRAP)
10623 {
10624 /*
10625 * The exception was for the guest. Update DR6, DR7.GD and
10626 * IA32_DEBUGCTL.LBR before forwarding it.
10627 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10628 */
10629 VMMRZCallRing3Disable(pVCpu);
10630 HM_DISABLE_PREEMPT_IF_NEEDED();
10631
10632 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10633 pMixedCtx->dr[6] |= uDR6;
10634 if (CPUMIsGuestDebugStateActive(pVCpu))
10635 ASMSetDR6(pMixedCtx->dr[6]);
10636
10637 HM_RESTORE_PREEMPT_IF_NEEDED();
10638 VMMRZCallRing3Enable(pVCpu);
10639
10640 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10641 AssertRCReturn(rc, rc);
10642
10643 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10644 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10645
10646 /* Paranoia. */
10647 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10648 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10649
10650 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10651 AssertRCReturn(rc, rc);
10652
10653 /*
10654 * Raise #DB in the guest.
10655 */
10656 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10657 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10658 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10659 AssertRCReturn(rc, rc);
10660 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10661 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10662 return VINF_SUCCESS;
10663 }
10664
10665 /*
10666 * Not a guest trap, must be a hypervisor related debug event then.
10667 * Update DR6 in case someone is interested in it.
10668 */
10669 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10670 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10671 CPUMSetHyperDR6(pVCpu, uDR6);
10672
10673 return rc;
10674}
10675
10676
10677/**
10678 * VM-exit exception handler for #NM (Device-not-available exception: floating
10679 * point exception).
10680 */
10681static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10682{
10683 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10684
10685 /* We require CR0 and EFER. EFER is always up-to-date. */
10686 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10687 AssertRCReturn(rc, rc);
10688
10689 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10690 VMMRZCallRing3Disable(pVCpu);
10691 HM_DISABLE_PREEMPT_IF_NEEDED();
10692
10693 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10694 if (pVmxTransient->fWasGuestFPUStateActive)
10695 {
10696 rc = VINF_EM_RAW_GUEST_TRAP;
10697 Assert(CPUMIsGuestFPUStateActive(pVCpu) || VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10698 }
10699 else
10700 {
10701#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10702 Assert(!pVmxTransient->fWasGuestFPUStateActive);
10703#endif
10704 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10705 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10706 }
10707
10708 HM_RESTORE_PREEMPT_IF_NEEDED();
10709 VMMRZCallRing3Enable(pVCpu);
10710
10711 if (rc == VINF_SUCCESS)
10712 {
10713 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10714 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10715 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10716 pVCpu->hm.s.fUseGuestFpu = true;
10717 }
10718 else
10719 {
10720 /* Forward #NM to the guest. */
10721 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10722 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10723 AssertRCReturn(rc, rc);
10724 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10725 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10727 }
10728
10729 return VINF_SUCCESS;
10730}
10731
10732
10733/**
10734 * VM-exit exception handler for #GP (General-protection exception).
10735 *
10736 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10737 */
10738static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10739{
10740 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10741 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10742
10743 int rc = VERR_INTERNAL_ERROR_5;
10744 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10745 {
10746#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10747 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10748 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10749 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10750 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10751 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10752 AssertRCReturn(rc, rc);
10753 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10754 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10755 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10756 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10757 return rc;
10758#else
10759 /* We don't intercept #GP. */
10760 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10761 return VERR_VMX_UNEXPECTED_EXCEPTION;
10762#endif
10763 }
10764
10765 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10766 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10767
10768 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10769 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10770 AssertRCReturn(rc, rc);
10771
10772 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10773 uint32_t cbOp = 0;
10774 PVM pVM = pVCpu->CTX_SUFF(pVM);
10775 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10776 if (RT_SUCCESS(rc))
10777 {
10778 rc = VINF_SUCCESS;
10779 Assert(cbOp == pDis->cbInstr);
10780 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10781 switch (pDis->pCurInstr->uOpcode)
10782 {
10783 case OP_CLI:
10784 {
10785 pMixedCtx->eflags.Bits.u1IF = 0;
10786 pMixedCtx->rip += pDis->cbInstr;
10787 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10788 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10789 break;
10790 }
10791
10792 case OP_STI:
10793 {
10794 pMixedCtx->eflags.Bits.u1IF = 1;
10795 pMixedCtx->rip += pDis->cbInstr;
10796 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10797 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10798 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10800 break;
10801 }
10802
10803 case OP_HLT:
10804 {
10805 rc = VINF_EM_HALT;
10806 pMixedCtx->rip += pDis->cbInstr;
10807 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10808 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10809 break;
10810 }
10811
10812 case OP_POPF:
10813 {
10814 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10815 uint32_t cbParm = 0;
10816 uint32_t uMask = 0;
10817 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10818 {
10819 cbParm = 4;
10820 uMask = 0xffffffff;
10821 }
10822 else
10823 {
10824 cbParm = 2;
10825 uMask = 0xffff;
10826 }
10827
10828 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10829 RTGCPTR GCPtrStack = 0;
10830 X86EFLAGS Eflags;
10831 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10832 &GCPtrStack);
10833 if (RT_SUCCESS(rc))
10834 {
10835 Assert(sizeof(Eflags.u32) >= cbParm);
10836 Eflags.u32 = 0;
10837 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10838 }
10839 if (RT_FAILURE(rc))
10840 {
10841 rc = VERR_EM_INTERPRETER;
10842 break;
10843 }
10844 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10845 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10846 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10847 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10848 pMixedCtx->esp += cbParm;
10849 pMixedCtx->esp &= uMask;
10850 pMixedCtx->rip += pDis->cbInstr;
10851
10852 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10853 | HM_CHANGED_GUEST_RSP
10854 | HM_CHANGED_GUEST_RFLAGS);
10855 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10856 break;
10857 }
10858
10859 case OP_PUSHF:
10860 {
10861 uint32_t cbParm = 0;
10862 uint32_t uMask = 0;
10863 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10864 {
10865 cbParm = 4;
10866 uMask = 0xffffffff;
10867 }
10868 else
10869 {
10870 cbParm = 2;
10871 uMask = 0xffff;
10872 }
10873
10874 /* Get the stack pointer & push the contents of eflags onto the stack. */
10875 RTGCPTR GCPtrStack = 0;
10876 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10877 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10878 if (RT_FAILURE(rc))
10879 {
10880 rc = VERR_EM_INTERPRETER;
10881 break;
10882 }
10883 X86EFLAGS Eflags = pMixedCtx->eflags;
10884 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10885 Eflags.Bits.u1RF = 0;
10886 Eflags.Bits.u1VM = 0;
10887
10888 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10889 if (RT_FAILURE(rc))
10890 {
10891 rc = VERR_EM_INTERPRETER;
10892 break;
10893 }
10894 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10895 pMixedCtx->esp -= cbParm;
10896 pMixedCtx->esp &= uMask;
10897 pMixedCtx->rip += pDis->cbInstr;
10898 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
10899 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10900 break;
10901 }
10902
10903 case OP_IRET:
10904 {
10905 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10906 * instruction reference. */
10907 RTGCPTR GCPtrStack = 0;
10908 uint32_t uMask = 0xffff;
10909 uint16_t aIretFrame[3];
10910 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10911 {
10912 rc = VERR_EM_INTERPRETER;
10913 break;
10914 }
10915 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10916 &GCPtrStack);
10917 if (RT_SUCCESS(rc))
10918 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10919 if (RT_FAILURE(rc))
10920 {
10921 rc = VERR_EM_INTERPRETER;
10922 break;
10923 }
10924 pMixedCtx->eip = 0;
10925 pMixedCtx->ip = aIretFrame[0];
10926 pMixedCtx->cs.Sel = aIretFrame[1];
10927 pMixedCtx->cs.ValidSel = aIretFrame[1];
10928 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10929 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10930 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10931 pMixedCtx->sp += sizeof(aIretFrame);
10932 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10933 | HM_CHANGED_GUEST_SEGMENT_REGS
10934 | HM_CHANGED_GUEST_RSP
10935 | HM_CHANGED_GUEST_RFLAGS);
10936 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10938 break;
10939 }
10940
10941 case OP_INT:
10942 {
10943 uint16_t uVector = pDis->Param1.uValue & 0xff;
10944 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10946 break;
10947 }
10948
10949 case OP_INTO:
10950 {
10951 if (pMixedCtx->eflags.Bits.u1OF)
10952 {
10953 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10955 }
10956 break;
10957 }
10958
10959 default:
10960 {
10961 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10962 EMCODETYPE_SUPERVISOR);
10963 rc = VBOXSTRICTRC_VAL(rc2);
10964 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10965 Log4(("#GP rc=%Rrc\n", rc));
10966 break;
10967 }
10968 }
10969 }
10970 else
10971 rc = VERR_EM_INTERPRETER;
10972
10973 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10974 ("#GP Unexpected rc=%Rrc\n", rc));
10975 return rc;
10976}
10977
10978
10979/**
10980 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10981 * the exception reported in the VMX transient structure back into the VM.
10982 *
10983 * @remarks Requires uExitIntInfo in the VMX transient structure to be
10984 * up-to-date.
10985 */
10986static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10987{
10988 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10989
10990 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10991 hmR0VmxCheckExitDueToEventDelivery(). */
10992 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10993 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10994 AssertRCReturn(rc, rc);
10995 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10996
10997 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10998 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10999 return VINF_SUCCESS;
11000}
11001
11002
11003/**
11004 * VM-exit exception handler for #PF (Page-fault exception).
11005 */
11006static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11007{
11008 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11009 PVM pVM = pVCpu->CTX_SUFF(pVM);
11010 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11011 rc |= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
11012 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11013 AssertRCReturn(rc, rc);
11014
11015#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11016 if (pVM->hm.s.fNestedPaging)
11017 {
11018 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11019 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11020 {
11021 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11022 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11023 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11024 }
11025 else
11026 {
11027 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11028 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11029 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11030 }
11031 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11032 return rc;
11033 }
11034#else
11035 Assert(!pVM->hm.s.fNestedPaging);
11036#endif
11037
11038 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11039 AssertRCReturn(rc, rc);
11040
11041 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11042 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11043
11044 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11045 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11046 (RTGCPTR)pVmxTransient->uExitQualification);
11047
11048 Log4(("#PF: rc=%Rrc\n", rc));
11049 if (rc == VINF_SUCCESS)
11050 {
11051 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11052 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11053 * memory? We don't update the whole state here... */
11054 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11055 | HM_CHANGED_GUEST_RSP
11056 | HM_CHANGED_GUEST_RFLAGS
11057 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11058 TRPMResetTrap(pVCpu);
11059 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11060 return rc;
11061 }
11062 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11063 {
11064 if (!pVmxTransient->fVectoringPF)
11065 {
11066 /* It's a guest page fault and needs to be reflected to the guest. */
11067 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11068 TRPMResetTrap(pVCpu);
11069 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11070 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11071 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11072 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11073 }
11074 else
11075 {
11076 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11077 TRPMResetTrap(pVCpu);
11078 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11079 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11080 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11081 }
11082
11083 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11084 return VINF_SUCCESS;
11085 }
11086
11087 TRPMResetTrap(pVCpu);
11088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11089 return rc;
11090}
11091
11092/** @} */
11093
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette