VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49258

Last change on this file since 49258 was 49258, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Introduced HMVMX_ALWAYS_FLUSH_TLB debug switch.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 460.8 KB
Line 
1/* $Id: HMVMXR0.cpp 49258 2013-10-23 16:10:21Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#define HMVMX_ALWAYS_FLUSH_TLB
46#endif
47
48
49/*******************************************************************************
50* Defined Constants And Macros *
51*******************************************************************************/
52#if defined(RT_ARCH_AMD64)
53# define HMVMX_IS_64BIT_HOST_MODE() (true)
54typedef RTHCUINTREG HMVMXHCUINTREG;
55#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56extern "C" uint32_t g_fVMXIs64bitHost;
57# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58typedef uint64_t HMVMXHCUINTREG;
59#else
60# define HMVMX_IS_64BIT_HOST_MODE() (false)
61typedef RTHCUINTREG HMVMXHCUINTREG;
62#endif
63
64/** Use the function table. */
65#define HMVMX_USE_FUNCTION_TABLE
66
67/**
68 * The maximum number of MSRs we are willing to swap during a world-switch.
69 * Intel claims 512/check capability MSR, we don't want to do anywhere close
70 * to that. See Intel spec. 24.7.2 "VM-Exit Controls for MSRs"
71 *
72 * Bump this count as and when required, there's no backward compatibility
73 * requirement.
74 */
75#define HMVMX_MAX_SWAP_MSR_COUNT 5
76
77/** Determine which tagged-TLB flush handler to use. */
78#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
79#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
80#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
81#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
82
83/** @name Updated-guest-state flags.
84 * @{ */
85#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
86#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
87#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
88#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
89#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
90#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
91#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
92#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
93#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
94#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
95#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
96#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
97#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
98#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
99#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
100#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
101#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
102#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
103#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
104#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
105#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
106 | HMVMX_UPDATED_GUEST_RSP \
107 | HMVMX_UPDATED_GUEST_RFLAGS \
108 | HMVMX_UPDATED_GUEST_CR0 \
109 | HMVMX_UPDATED_GUEST_CR3 \
110 | HMVMX_UPDATED_GUEST_CR4 \
111 | HMVMX_UPDATED_GUEST_GDTR \
112 | HMVMX_UPDATED_GUEST_IDTR \
113 | HMVMX_UPDATED_GUEST_LDTR \
114 | HMVMX_UPDATED_GUEST_TR \
115 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
116 | HMVMX_UPDATED_GUEST_DEBUG \
117 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
118 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
119 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
120 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
121 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
122 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
123 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
124 | HMVMX_UPDATED_GUEST_APIC_STATE)
125/** @} */
126
127/** @name
128 * Flags to skip redundant reads of some common VMCS fields that are not part of
129 * the guest-CPU state but are in the transient structure.
130 */
131#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
132#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
133#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
134#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
135#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
136#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
137#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
138/** @} */
139
140/** @name
141 * States of the VMCS.
142 *
143 * This does not reflect all possible VMCS states but currently only those
144 * needed for maintaining the VMCS consistently even when thread-context hooks
145 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
146 */
147#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
148#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
149#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
150/** @} */
151
152/**
153 * Exception bitmap mask for real-mode guests (real-on-v86).
154 *
155 * We need to intercept all exceptions manually (except #PF). #NM is also
156 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
157 * even in real-mode if we have Nested Paging support.
158 */
159#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
160 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
161 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
162 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
163 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
164 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
165 | RT_BIT(X86_XCPT_XF))
166
167/**
168 * Exception bitmap mask for all contributory exceptions.
169 *
170 * Page fault is deliberately excluded here as it's conditional as to whether
171 * it's contributory or benign. Page faults are handled separately.
172 */
173#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
174 | RT_BIT(X86_XCPT_DE))
175
176/** Maximum VM-instruction error number. */
177#define HMVMX_INSTR_ERROR_MAX 28
178
179/** Profiling macro. */
180#ifdef HM_PROFILE_EXIT_DISPATCH
181# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
183#else
184# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
185# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
186#endif
187
188/** Assert that preemption is disabled or covered by thread-context hooks. */
189#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
191
192/** Assert that we haven't migrated CPUs when thread-context hooks are not
193 * used. */
194#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
195 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
196 ("Illegal migration! Entered on CPU %u Current %u\n", \
197 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
198
199/** Helper macro for VM-exit handlers called unexpectedly. */
200#define HMVMX_RETURN_UNEXPECTED_EXIT() \
201 do { \
202 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
203 return VERR_VMX_UNEXPECTED_EXIT; \
204 } while (0)
205
206
207/*******************************************************************************
208* Structures and Typedefs *
209*******************************************************************************/
210/**
211 * VMX transient state.
212 *
213 * A state structure for holding miscellaneous information across
214 * VMX non-root operation and restored after the transition.
215 */
216typedef struct VMXTRANSIENT
217{
218 /** The host's rflags/eflags. */
219 RTCCUINTREG uEflags;
220#if HC_ARCH_BITS == 32
221 uint32_t u32Alignment0;
222#endif
223 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
224 uint64_t u64LStarMsr;
225 /** The guest's TPR value used for TPR shadowing. */
226 uint8_t u8GuestTpr;
227 /** Alignment. */
228 uint8_t abAlignment0[7];
229
230 /** The basic VM-exit reason. */
231 uint16_t uExitReason;
232 /** Alignment. */
233 uint16_t u16Alignment0;
234 /** The VM-exit interruption error code. */
235 uint32_t uExitIntErrorCode;
236 /** The VM-exit exit qualification. */
237 uint64_t uExitQualification;
238
239 /** The VM-exit interruption-information field. */
240 uint32_t uExitIntInfo;
241 /** The VM-exit instruction-length field. */
242 uint32_t cbInstr;
243 /** The VM-exit instruction-information field. */
244 union
245 {
246 /** Plain unsigned int representation. */
247 uint32_t u;
248 /** INS and OUTS information. */
249 struct
250 {
251 uint32_t u6Reserved0 : 7;
252 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253 uint32_t u3AddrSize : 3;
254 uint32_t u5Reserved1 : 5;
255 /** The segment register (X86_SREG_XXX). */
256 uint32_t iSegReg : 3;
257 uint32_t uReserved2 : 14;
258 } StrIo;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest FPU was active at the time of VM-exit. */
281 bool fWasGuestFPUStateActive;
282 /** Whether the guest debug state was active at the time of VM-exit. */
283 bool fWasGuestDebugStateActive;
284 /** Whether the hyper debug state was active at the time of VM-exit. */
285 bool fWasHyperDebugStateActive;
286 /** Whether TSC-offsetting should be setup before VM-entry. */
287 bool fUpdateTscOffsettingAndPreemptTimer;
288 /** Whether the VM-exit was caused by a page-fault during delivery of a
289 * contributory exception or a page-fault. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322
323/**
324 * VMX VM-exit handler.
325 *
326 * @returns VBox status code.
327 * @param pVCpu Pointer to the VMCPU.
328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
329 * out-of-sync. Make sure to update the required
330 * fields before using them.
331 * @param pVmxTransient Pointer to the VMX-transient structure.
332 */
333#ifndef HMVMX_USE_FUNCTION_TABLE
334typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335#else
336typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
337/** Pointer to VM-exit handler. */
338typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
339#endif
340
341
342/*******************************************************************************
343* Internal Functions *
344*******************************************************************************/
345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
346static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
347static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
348static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
350#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
351static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
352#endif
353#ifndef HMVMX_USE_FUNCTION_TABLE
354DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
355# define HMVMX_EXIT_DECL static int
356#else
357# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
358#endif
359
360/** @name VM-exit handlers.
361 * @{
362 */
363static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
364static FNVMXEXITHANDLER hmR0VmxExitExtInt;
365static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
366static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
367static FNVMXEXITHANDLER hmR0VmxExitSipi;
368static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
369static FNVMXEXITHANDLER hmR0VmxExitSmi;
370static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
371static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
372static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
373static FNVMXEXITHANDLER hmR0VmxExitCpuid;
374static FNVMXEXITHANDLER hmR0VmxExitGetsec;
375static FNVMXEXITHANDLER hmR0VmxExitHlt;
376static FNVMXEXITHANDLER hmR0VmxExitInvd;
377static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
378static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
379static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
380static FNVMXEXITHANDLER hmR0VmxExitRsm;
381static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
382static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
383static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
384static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
385static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
386static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
387static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
388static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
389static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
390static FNVMXEXITHANDLER hmR0VmxExitMwait;
391static FNVMXEXITHANDLER hmR0VmxExitMtf;
392static FNVMXEXITHANDLER hmR0VmxExitMonitor;
393static FNVMXEXITHANDLER hmR0VmxExitPause;
394static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
395static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
396static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
397static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
398static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
404static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407/** @} */
408
409static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
417
418/*******************************************************************************
419* Global Variables *
420*******************************************************************************/
421#ifdef HMVMX_USE_FUNCTION_TABLE
422
423/**
424 * VMX_EXIT dispatch table.
425 */
426static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
427{
428 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
429 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
430 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
431 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
432 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
433 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
434 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
435 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
436 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
437 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
438 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
439 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
440 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
441 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
442 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
443 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
444 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
445 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
446 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
447 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
448 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
449 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
450 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
451 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
452 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
453 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
454 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
455 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
456 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
457 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
458 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
459 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
460 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
461 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
462 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
463 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
464 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
465 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
466 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
468 /* 40 UNDEFINED */ hmR0VmxExitPause,
469 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
470 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
471 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
472 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
473 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
474 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
475 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
476 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
477 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
478 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
479 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
480 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
481 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
482 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
483 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
484 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
485 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
486 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
487 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
488};
489#endif /* HMVMX_USE_FUNCTION_TABLE */
490
491#ifdef VBOX_STRICT
492static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
493{
494 /* 0 */ "(Not Used)",
495 /* 1 */ "VMCALL executed in VMX root operation.",
496 /* 2 */ "VMCLEAR with invalid physical address.",
497 /* 3 */ "VMCLEAR with VMXON pointer.",
498 /* 4 */ "VMLAUNCH with non-clear VMCS.",
499 /* 5 */ "VMRESUME with non-launched VMCS.",
500 /* 6 */ "VMRESUME after VMXOFF",
501 /* 7 */ "VM entry with invalid control fields.",
502 /* 8 */ "VM entry with invalid host state fields.",
503 /* 9 */ "VMPTRLD with invalid physical address.",
504 /* 10 */ "VMPTRLD with VMXON pointer.",
505 /* 11 */ "VMPTRLD with incorrect revision identifier.",
506 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
507 /* 13 */ "VMWRITE to read-only VMCS component.",
508 /* 14 */ "(Not Used)",
509 /* 15 */ "VMXON executed in VMX root operation.",
510 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
511 /* 17 */ "VM entry with non-launched executing VMCS.",
512 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
513 /* 19 */ "VMCALL with non-clear VMCS.",
514 /* 20 */ "VMCALL with invalid VM-exit control fields.",
515 /* 21 */ "(Not Used)",
516 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
517 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
518 /* 24 */ "VMCALL with invalid SMM-monitor features.",
519 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
520 /* 26 */ "VM entry with events blocked by MOV SS.",
521 /* 27 */ "(Not Used)",
522 /* 28 */ "Invalid operand to INVEPT/INVVPID."
523};
524#endif /* VBOX_STRICT */
525
526
527
528/**
529 * Updates the VM's last error record. If there was a VMX instruction error,
530 * reads the error data from the VMCS and updates VCPU's last error record as
531 * well.
532 *
533 * @param pVM Pointer to the VM.
534 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
535 * VERR_VMX_UNABLE_TO_START_VM or
536 * VERR_VMX_INVALID_VMCS_FIELD).
537 * @param rc The error code.
538 */
539static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
540{
541 AssertPtr(pVM);
542 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
543 || rc == VERR_VMX_UNABLE_TO_START_VM)
544 {
545 AssertPtrReturnVoid(pVCpu);
546 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
547 }
548 pVM->hm.s.lLastError = rc;
549}
550
551
552/**
553 * Reads the VM-entry interruption-information field from the VMCS into the VMX
554 * transient structure.
555 *
556 * @returns VBox status code.
557 * @param pVmxTransient Pointer to the VMX transient structure.
558 *
559 * @remarks No-long-jump zone!!!
560 */
561DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
562{
563 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
564 AssertRCReturn(rc, rc);
565 return VINF_SUCCESS;
566}
567
568
569/**
570 * Reads the VM-entry exception error code field from the VMCS into
571 * the VMX transient structure.
572 *
573 * @returns VBox status code.
574 * @param pVmxTransient Pointer to the VMX transient structure.
575 *
576 * @remarks No-long-jump zone!!!
577 */
578DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
579{
580 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
581 AssertRCReturn(rc, rc);
582 return VINF_SUCCESS;
583}
584
585
586/**
587 * Reads the VM-entry exception error code field from the VMCS into
588 * the VMX transient structure.
589 *
590 * @returns VBox status code.
591 * @param pVCpu Pointer to the VMCPU.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-exit interruption-information field from the VMCS into the VMX
606 * transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVCpu Pointer to the VMCPU.
610 * @param pVmxTransient Pointer to the VMX transient structure.
611 */
612DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
613{
614 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
615 {
616 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
617 AssertRCReturn(rc, rc);
618 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
619 }
620 return VINF_SUCCESS;
621}
622
623
624/**
625 * Reads the VM-exit interruption error code from the VMCS into the VMX
626 * transient structure.
627 *
628 * @returns VBox status code.
629 * @param pVCpu Pointer to the VMCPU.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 */
632DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
633{
634 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
635 {
636 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
637 AssertRCReturn(rc, rc);
638 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
639 }
640 return VINF_SUCCESS;
641}
642
643
644/**
645 * Reads the VM-exit instruction length field from the VMCS into the VMX
646 * transient structure.
647 *
648 * @returns VBox status code.
649 * @param pVCpu Pointer to the VMCPU.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
657 AssertRCReturn(rc, rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit instruction-information field from the VMCS into
666 * the VMX transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVCpu The cross context per CPU structure.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit qualification from the VMCS into the VMX transient structure.
686 *
687 * @returns VBox status code.
688 * @param pVCpu Pointer to the VMCPU.
689 * @param pVmxTransient Pointer to the VMX transient structure.
690 */
691DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
692{
693 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
694 {
695 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
696 AssertRCReturn(rc, rc);
697 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
698 }
699 return VINF_SUCCESS;
700}
701
702
703/**
704 * Reads the IDT-vectoring information field from the VMCS into the VMX
705 * transient structure.
706 *
707 * @returns VBox status code.
708 * @param pVmxTransient Pointer to the VMX transient structure.
709 *
710 * @remarks No-long-jump zone!!!
711 */
712DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
713{
714 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
715 {
716 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
717 AssertRCReturn(rc, rc);
718 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
719 }
720 return VINF_SUCCESS;
721}
722
723
724/**
725 * Reads the IDT-vectoring error code from the VMCS into the VMX
726 * transient structure.
727 *
728 * @returns VBox status code.
729 * @param pVmxTransient Pointer to the VMX transient structure.
730 */
731DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
732{
733 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
734 {
735 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
736 AssertRCReturn(rc, rc);
737 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
738 }
739 return VINF_SUCCESS;
740}
741
742
743/**
744 * Enters VMX root mode operation on the current CPU.
745 *
746 * @returns VBox status code.
747 * @param pVM Pointer to the VM (optional, can be NULL, after
748 * a resume).
749 * @param HCPhysCpuPage Physical address of the VMXON region.
750 * @param pvCpuPage Pointer to the VMXON region.
751 */
752static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
753{
754 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
755 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 if (pVM)
759 {
760 /* Write the VMCS revision dword to the VMXON region. */
761 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
762 }
763
764 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
765 RTCCUINTREG uEflags = ASMIntDisableFlags();
766
767 /* Enable the VMX bit in CR4 if necessary. */
768 RTCCUINTREG uCr4 = ASMGetCR4();
769 if (!(uCr4 & X86_CR4_VMXE))
770 ASMSetCR4(uCr4 | X86_CR4_VMXE);
771
772 /* Enter VMX root mode. */
773 int rc = VMXEnable(HCPhysCpuPage);
774 if (RT_FAILURE(rc))
775 ASMSetCR4(uCr4);
776
777 /* Restore interrupts. */
778 ASMSetFlags(uEflags);
779 return rc;
780}
781
782
783/**
784 * Exits VMX root mode operation on the current CPU.
785 *
786 * @returns VBox status code.
787 */
788static int hmR0VmxLeaveRootMode(void)
789{
790 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
791
792 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
793 RTCCUINTREG uEflags = ASMIntDisableFlags();
794
795 /* If we're for some reason not in VMX root mode, then don't leave it. */
796 RTCCUINTREG uHostCR4 = ASMGetCR4();
797
798 int rc;
799 if (uHostCR4 & X86_CR4_VMXE)
800 {
801 /* Exit VMX root mode and clear the VMX bit in CR4. */
802 VMXDisable();
803 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
804 rc = VINF_SUCCESS;
805 }
806 else
807 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
808
809 /* Restore interrupts. */
810 ASMSetFlags(uEflags);
811 return rc;
812}
813
814
815/**
816 * Allocates and maps one physically contiguous page. The allocated page is
817 * zero'd out. (Used by various VT-x structures).
818 *
819 * @returns IPRT status code.
820 * @param pMemObj Pointer to the ring-0 memory object.
821 * @param ppVirt Where to store the virtual address of the
822 * allocation.
823 * @param pPhys Where to store the physical address of the
824 * allocation.
825 */
826DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
827{
828 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
829 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
830 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
831
832 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
833 if (RT_FAILURE(rc))
834 return rc;
835 *ppVirt = RTR0MemObjAddress(*pMemObj);
836 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
837 ASMMemZero32(*ppVirt, PAGE_SIZE);
838 return VINF_SUCCESS;
839}
840
841
842/**
843 * Frees and unmaps an allocated physical page.
844 *
845 * @param pMemObj Pointer to the ring-0 memory object.
846 * @param ppVirt Where to re-initialize the virtual address of
847 * allocation as 0.
848 * @param pHCPhys Where to re-initialize the physical address of the
849 * allocation as 0.
850 */
851DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
852{
853 AssertPtr(pMemObj);
854 AssertPtr(ppVirt);
855 AssertPtr(pHCPhys);
856 if (*pMemObj != NIL_RTR0MEMOBJ)
857 {
858 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
859 AssertRC(rc);
860 *pMemObj = NIL_RTR0MEMOBJ;
861 *ppVirt = 0;
862 *pHCPhys = 0;
863 }
864}
865
866
867/**
868 * Worker function to free VT-x related structures.
869 *
870 * @returns IPRT status code.
871 * @param pVM Pointer to the VM.
872 */
873static void hmR0VmxStructsFree(PVM pVM)
874{
875 for (VMCPUID i = 0; i < pVM->cCpus; i++)
876 {
877 PVMCPU pVCpu = &pVM->aCpus[i];
878 AssertPtr(pVCpu);
879
880#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
881 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
883#endif
884
885 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
887
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
890 }
891
892 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
893#ifdef VBOX_WITH_CRASHDUMP_MAGIC
894 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
895#endif
896}
897
898
899/**
900 * Worker function to allocate VT-x related VM structures.
901 *
902 * @returns IPRT status code.
903 * @param pVM Pointer to the VM.
904 */
905static int hmR0VmxStructsAlloc(PVM pVM)
906{
907 /*
908 * Initialize members up-front so we can cleanup properly on allocation failure.
909 */
910#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
911 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
912 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
913 pVM->hm.s.vmx.HCPhys##a_Name = 0;
914
915#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
919
920#ifdef VBOX_WITH_CRASHDUMP_MAGIC
921 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
922#endif
923 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
924
925 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
926 for (VMCPUID i = 0; i < pVM->cCpus; i++)
927 {
928 PVMCPU pVCpu = &pVM->aCpus[i];
929 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
931 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
932#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935#endif
936 }
937#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
938#undef VMXLOCAL_INIT_VM_MEMOBJ
939
940 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
941 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
942 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
943 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
944
945 /*
946 * Allocate all the VT-x structures.
947 */
948 int rc = VINF_SUCCESS;
949#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951 if (RT_FAILURE(rc))
952 goto cleanup;
953 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
954 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
955#endif
956
957 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
958 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
959 {
960 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
961 &pVM->hm.s.vmx.HCPhysApicAccess);
962 if (RT_FAILURE(rc))
963 goto cleanup;
964 }
965
966 /*
967 * Initialize per-VCPU VT-x structures.
968 */
969 for (VMCPUID i = 0; i < pVM->cCpus; i++)
970 {
971 PVMCPU pVCpu = &pVM->aCpus[i];
972 AssertPtr(pVCpu);
973
974 /* Allocate the VM control structure (VMCS). */
975 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978
979 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
980 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
981 {
982 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
983 &pVCpu->hm.s.vmx.HCPhysVirtApic);
984 if (RT_FAILURE(rc))
985 goto cleanup;
986 }
987
988 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
990 {
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
992 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
996 }
997
998#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
999 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003
1004 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1005 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008#endif
1009 }
1010
1011 return VINF_SUCCESS;
1012
1013cleanup:
1014 hmR0VmxStructsFree(pVM);
1015 return rc;
1016}
1017
1018
1019/**
1020 * Does global VT-x initialization (called during module initialization).
1021 *
1022 * @returns VBox status code.
1023 */
1024VMMR0DECL(int) VMXR0GlobalInit(void)
1025{
1026#ifdef HMVMX_USE_FUNCTION_TABLE
1027 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1028# ifdef VBOX_STRICT
1029 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1030 Assert(g_apfnVMExitHandlers[i]);
1031# endif
1032#endif
1033 return VINF_SUCCESS;
1034}
1035
1036
1037/**
1038 * Does global VT-x termination (called during module termination).
1039 */
1040VMMR0DECL(void) VMXR0GlobalTerm()
1041{
1042 /* Nothing to do currently. */
1043}
1044
1045
1046/**
1047 * Sets up and activates VT-x on the current CPU.
1048 *
1049 * @returns VBox status code.
1050 * @param pCpu Pointer to the global CPU info struct.
1051 * @param pVM Pointer to the VM (can be NULL after a host resume
1052 * operation).
1053 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1054 * fEnabledByHost is true).
1055 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1056 * @a fEnabledByHost is true).
1057 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1058 * enable VT-x on the host.
1059 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1060 */
1061VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1062 void *pvMsrs)
1063{
1064 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1065 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067
1068 /* Enable VT-x if it's not already enabled by the host. */
1069 if (!fEnabledByHost)
1070 {
1071 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1072 if (RT_FAILURE(rc))
1073 return rc;
1074 }
1075
1076 /*
1077 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1078 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1079 */
1080 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1081 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1082 {
1083 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1084 pCpu->fFlushAsidBeforeUse = false;
1085 }
1086 else
1087 pCpu->fFlushAsidBeforeUse = true;
1088
1089 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1090 ++pCpu->cTlbFlushes;
1091
1092 return VINF_SUCCESS;
1093}
1094
1095
1096/**
1097 * Deactivates VT-x on the current CPU.
1098 *
1099 * @returns VBox status code.
1100 * @param pCpu Pointer to the global CPU info struct.
1101 * @param pvCpuPage Pointer to the VMXON region.
1102 * @param HCPhysCpuPage Physical address of the VMXON region.
1103 *
1104 * @remarks This function should never be called when SUPR0EnableVTx() or
1105 * similar was used to enable VT-x on the host.
1106 */
1107VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1108{
1109 NOREF(pCpu);
1110 NOREF(pvCpuPage);
1111 NOREF(HCPhysCpuPage);
1112
1113 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1114 return hmR0VmxLeaveRootMode();
1115}
1116
1117
1118/**
1119 * Sets the permission bits for the specified MSR in the MSR bitmap.
1120 *
1121 * @param pVCpu Pointer to the VMCPU.
1122 * @param uMSR The MSR value.
1123 * @param enmRead Whether reading this MSR causes a VM-exit.
1124 * @param enmWrite Whether writing this MSR causes a VM-exit.
1125 */
1126static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1127{
1128 int32_t iBit;
1129 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1130
1131 /*
1132 * Layout:
1133 * 0x000 - 0x3ff - Low MSR read bits
1134 * 0x400 - 0x7ff - High MSR read bits
1135 * 0x800 - 0xbff - Low MSR write bits
1136 * 0xc00 - 0xfff - High MSR write bits
1137 */
1138 if (uMsr <= 0x00001FFF)
1139 iBit = uMsr;
1140 else if ( uMsr >= 0xC0000000
1141 && uMsr <= 0xC0001FFF)
1142 {
1143 iBit = (uMsr - 0xC0000000);
1144 pbMsrBitmap += 0x400;
1145 }
1146 else
1147 {
1148 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1149 return;
1150 }
1151
1152 Assert(iBit <= 0x1fff);
1153 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154 ASMBitSet(pbMsrBitmap, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap, iBit);
1157
1158 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160 else
1161 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162}
1163
1164
1165#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1166/**
1167 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1168 * area.
1169 *
1170 * @returns VBox status code.
1171 * @param pVCpu Pointer to the VMCPU.
1172 * @param cMsrs The number of MSRs.
1173 */
1174DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1175{
1176 /* Update the VCPU's copy of the guest MSR count. */
1177 pVCpu->hm.s.vmx.cGuestMsrs = cMsrs;
1178
1179 /* Update number of guest MSRs to load/store across the world-switch. */
1180 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1181 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1182
1183 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1184 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1185 return VINF_SUCCESS;
1186}
1187
1188
1189/**
1190 * Adds a guest/host MSR pair to be swapped during the world-switch as
1191 * part of the auto-load/store MSR area in the VMCS.
1192 *
1193 * @returns VBox status code.
1194 * @param pVCpu Pointer to the VMCPU.
1195 * @param uMsr The MSR.
1196 * @param uGuestMsr Value of the guest MSR.
1197 * @param uHostMsr Value of the host MSR.
1198 */
1199static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, uint64_t uHostMsrValue)
1200{
1201 AssertMsg(HMVMX_MAX_SWAP_MSR_COUNT < MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc),
1202 ("MSR swap count exceeded. Cpu reports %#RX32, our limit %#RX32\n",
1203 MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc), HMVMX_MAX_SWAP_MSR_COUNT));
1204
1205 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1206 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1207 uint32_t i;
1208 for (i = 0; i < cGuestMsrs; i++)
1209 {
1210 if (pGuestMsr->u32Msr == uMsr)
1211 break;
1212 pGuestMsr++;
1213 }
1214
1215 AssertReturn(i < HMVMX_MAX_SWAP_MSR_COUNT, VERR_HM_MSR_SWAP_COUNT_EXCEEDED);
1216 if (i == cGuestMsrs)
1217 {
1218 ++cGuestMsrs;
1219 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1220 if (RT_UNLIKELY(cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc)))
1221 {
1222 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
1223 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1224 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1225 }
1226
1227 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1228 AssertRCReturn(rc, rc);
1229 }
1230
1231 /* Update the MSR values in the auto-load/store MSR area. */
1232 pGuestMsr->u32Msr = uMsr;
1233 pGuestMsr->u64Value = uGuestMsrValue;
1234
1235 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1236 pHostMsr += i;
1237 pHostMsr->u32Msr = uMsr;
1238 pHostMsr->u64Value = uHostMsrValue;
1239
1240 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1241 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
1242 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1243 return VINF_SUCCESS;
1244}
1245
1246
1247/**
1248 * Removes a guest/shost MSR pair to be swapped during the world-switch from the
1249 * auto-load/store MSR area in the VMCS.
1250 *
1251 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1252 * area.
1253 *
1254 * @returns VBox status code.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 */
1258static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cGuestMsrs; i++)
1264 {
1265 /* Find the MSR. */
1266 if (pGuestMsr->u32Msr == uMsr)
1267 {
1268 /* If it's the last MSR, simply reduce the count. */
1269 if (i == cGuestMsrs - 1)
1270 {
1271 --cGuestMsrs;
1272 break;
1273 }
1274
1275 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1276 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1277 pLastGuestMsr += cGuestMsrs;
1278 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1279 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1280
1281 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1282 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1283 pLastHostMsr += cGuestMsrs;
1284 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1285 pHostMsr->u64Value = pLastHostMsr->u64Value;
1286 --cGuestMsrs;
1287 break;
1288 }
1289 pGuestMsr++;
1290 }
1291
1292 /* Update the VMCS if the count changed (meaning the MSR was found). */
1293 if (cGuestMsrs != pVCpu->hm.s.vmx.cGuestMsrs)
1294 {
1295 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1296 AssertRCReturn(rc, rc);
1297 }
1298
1299 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1300 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1301 return VINF_SUCCESS;
1302}
1303
1304
1305/**
1306 * Updates the value of a host MSR in the auto-load/store area in the VMCS.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxUpdateAutoLoadStoreHostMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1316
1317 for (uint32_t i = 0; i < cMsrs; i++)
1318 {
1319 if (pHostMsr->u32Msr == uMsr)
1320 {
1321 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1322 return VINF_SUCCESS;
1323 }
1324 }
1325
1326 return VERR_NOT_FOUND;
1327}
1328
1329#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1330
1331
1332/**
1333 * Flushes the TLB using EPT.
1334 *
1335 * @returns VBox status code.
1336 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1337 * enmFlush).
1338 * @param enmFlush Type of flush.
1339 *
1340 * @remarks Caller is responsible for making sure this function is called only
1341 * when NestedPaging is supported and providing @a enmFlush that is
1342 * supported by the CPU.
1343 * @remarks Can be called with interrupts disabled.
1344 */
1345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1346{
1347 uint64_t au64Descriptor[2];
1348 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1349 au64Descriptor[0] = 0;
1350 else
1351 {
1352 Assert(pVCpu);
1353 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1354 }
1355 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1356
1357 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1358 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1359 rc));
1360 if ( RT_SUCCESS(rc)
1361 && pVCpu)
1362 {
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1364 }
1365}
1366
1367
1368/**
1369 * Flushes the TLB using VPID.
1370 *
1371 * @returns VBox status code.
1372 * @param pVM Pointer to the VM.
1373 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1374 * enmFlush).
1375 * @param enmFlush Type of flush.
1376 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1377 * on @a enmFlush).
1378 *
1379 * @remarks Can be called with interrupts disabled.
1380 */
1381static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1382{
1383 AssertPtr(pVM);
1384 Assert(pVM->hm.s.vmx.fVpid);
1385
1386 uint64_t au64Descriptor[2];
1387 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1388 {
1389 au64Descriptor[0] = 0;
1390 au64Descriptor[1] = 0;
1391 }
1392 else
1393 {
1394 AssertPtr(pVCpu);
1395 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1396 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1397 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1398 au64Descriptor[1] = GCPtr;
1399 }
1400
1401 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1402 AssertMsg(rc == VINF_SUCCESS,
1403 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1404 if ( RT_SUCCESS(rc)
1405 && pVCpu)
1406 {
1407 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1408 }
1409}
1410
1411
1412/**
1413 * Invalidates a guest page by guest virtual address. Only relevant for
1414 * EPT/VPID, otherwise there is nothing really to invalidate.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param GCVirt Guest virtual address of the page to invalidate.
1420 */
1421VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1422{
1423 AssertPtr(pVM);
1424 AssertPtr(pVCpu);
1425 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1426
1427 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1428 if (!fFlushPending)
1429 {
1430 /*
1431 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1432 * See @bugref{6043} and @bugref{6177}.
1433 *
1434 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1435 * function maybe called in a loop with individual addresses.
1436 */
1437 if (pVM->hm.s.vmx.fVpid)
1438 {
1439 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1440 {
1441 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1442 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1443 }
1444 else
1445 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1446 }
1447 else if (pVM->hm.s.fNestedPaging)
1448 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1449 }
1450
1451 return VINF_SUCCESS;
1452}
1453
1454
1455/**
1456 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1457 * otherwise there is nothing really to invalidate.
1458 *
1459 * @returns VBox status code.
1460 * @param pVM Pointer to the VM.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param GCPhys Guest physical address of the page to invalidate.
1463 */
1464VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1465{
1466 LogFlowFunc(("%RGp\n", GCPhys));
1467
1468 /*
1469 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1470 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1471 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1472 */
1473 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1474 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1475 return VINF_SUCCESS;
1476}
1477
1478
1479/**
1480 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1481 * case where neither EPT nor VPID is supported by the CPU.
1482 *
1483 * @param pVM Pointer to the VM.
1484 * @param pVCpu Pointer to the VMCPU.
1485 * @param pCpu Pointer to the global HM struct.
1486 *
1487 * @remarks Called with interrupts disabled.
1488 */
1489static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1490{
1491 AssertPtr(pVCpu);
1492 AssertPtr(pCpu);
1493 NOREF(pVM);
1494
1495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1496
1497 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1498#if 0
1499 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1500 pVCpu->hm.s.TlbShootdown.cPages = 0;
1501#endif
1502
1503 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1504 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1505 pVCpu->hm.s.fForceTLBFlush = false;
1506 return;
1507}
1508
1509
1510/**
1511 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1512 *
1513 * @param pVM Pointer to the VM.
1514 * @param pVCpu Pointer to the VMCPU.
1515 * @param pCpu Pointer to the global HM CPU struct.
1516 * @remarks All references to "ASID" in this function pertains to "VPID" in
1517 * Intel's nomenclature. The reason is, to avoid confusion in compare
1518 * statements since the host-CPU copies are named "ASID".
1519 *
1520 * @remarks Called with interrupts disabled.
1521 */
1522static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1523{
1524#ifdef VBOX_WITH_STATISTICS
1525 bool fTlbFlushed = false;
1526# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1527# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1528 if (!fTlbFlushed) \
1529 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1530 } while (0)
1531#else
1532# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1533# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1534#endif
1535
1536 AssertPtr(pVM);
1537 AssertPtr(pCpu);
1538 AssertPtr(pVCpu);
1539 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1540 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1541 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1542
1543
1544 /*
1545 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1546 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1547 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1548 */
1549 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1550 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1551 {
1552 ++pCpu->uCurrentAsid;
1553 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1554 {
1555 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1556 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1557 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1558 }
1559
1560 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1561 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1562 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1563
1564 /*
1565 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1566 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1567 */
1568 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1569 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1570 HMVMX_SET_TAGGED_TLB_FLUSHED();
1571 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1572 }
1573
1574 /* Check for explicit TLB shootdowns. */
1575 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1576 {
1577 /*
1578 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1579 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1580 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1581 * but not guest-physical mappings.
1582 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1583 */
1584 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1585 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1586 HMVMX_SET_TAGGED_TLB_FLUSHED();
1587 }
1588
1589 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1590 * where it is commented out. Support individual entry flushing
1591 * someday. */
1592#if 0
1593 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1594 {
1595 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1596
1597 /*
1598 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1599 * as supported by the CPU.
1600 */
1601 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1602 {
1603 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1604 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1605 }
1606 else
1607 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1608
1609 HMVMX_SET_TAGGED_TLB_FLUSHED();
1610 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1611 pVCpu->hm.s.TlbShootdown.cPages = 0;
1612 }
1613#endif
1614
1615 pVCpu->hm.s.fForceTLBFlush = false;
1616
1617 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1618
1619 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1620 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1621 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1622 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1623 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1624 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1625 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1626 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1627
1628 /* Update VMCS with the VPID. */
1629 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1630 AssertRC(rc);
1631
1632#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1633}
1634
1635
1636/**
1637 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1638 *
1639 * @returns VBox status code.
1640 * @param pVM Pointer to the VM.
1641 * @param pVCpu Pointer to the VMCPU.
1642 * @param pCpu Pointer to the global HM CPU struct.
1643 *
1644 * @remarks Called with interrupts disabled.
1645 */
1646static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1647{
1648 AssertPtr(pVM);
1649 AssertPtr(pVCpu);
1650 AssertPtr(pCpu);
1651 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1652 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1653
1654 /*
1655 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1656 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1657 */
1658 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1659 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1660 {
1661 pVCpu->hm.s.fForceTLBFlush = true;
1662 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1663 }
1664
1665 /* Check for explicit TLB shootdown flushes. */
1666 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1667 {
1668 pVCpu->hm.s.fForceTLBFlush = true;
1669 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1670 }
1671
1672 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1673 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1674
1675 if (pVCpu->hm.s.fForceTLBFlush)
1676 {
1677 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1678 pVCpu->hm.s.fForceTLBFlush = false;
1679 }
1680 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1681 * where it is commented out. Support individual entry flushing
1682 * someday. */
1683#if 0
1684 else
1685 {
1686 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1687 {
1688 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1689 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1690 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1691 }
1692 else
1693 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1694
1695 pVCpu->hm.s.TlbShootdown.cPages = 0;
1696 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1697 }
1698#endif
1699}
1700
1701
1702/**
1703 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1704 *
1705 * @returns VBox status code.
1706 * @param pVM Pointer to the VM.
1707 * @param pVCpu Pointer to the VMCPU.
1708 * @param pCpu Pointer to the global HM CPU struct.
1709 *
1710 * @remarks Called with interrupts disabled.
1711 */
1712static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1713{
1714 AssertPtr(pVM);
1715 AssertPtr(pVCpu);
1716 AssertPtr(pCpu);
1717 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1718 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1719
1720 /*
1721 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1722 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1723 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1724 */
1725 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1726 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1727 {
1728 pVCpu->hm.s.fForceTLBFlush = true;
1729 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1730 }
1731
1732 /* Check for explicit TLB shootdown flushes. */
1733 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1734 {
1735 /*
1736 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1737 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1738 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1739 */
1740 pVCpu->hm.s.fForceTLBFlush = true;
1741 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1742 }
1743
1744 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1745 if (pVCpu->hm.s.fForceTLBFlush)
1746 {
1747 ++pCpu->uCurrentAsid;
1748 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1749 {
1750 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1751 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1752 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1753 }
1754
1755 pVCpu->hm.s.fForceTLBFlush = false;
1756 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1757 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1758 if (pCpu->fFlushAsidBeforeUse)
1759 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1760 }
1761 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1762 * where it is commented out. Support individual entry flushing
1763 * someday. */
1764#if 0
1765 else
1766 {
1767 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1768 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1769 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1770 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1771
1772 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1773 {
1774 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1776 {
1777 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1778 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1779 }
1780 else
1781 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1782
1783 pVCpu->hm.s.TlbShootdown.cPages = 0;
1784 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1785 }
1786 else
1787 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1788 }
1789#endif
1790
1791 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1792 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1793 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1794 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1795 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1796 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1797
1798 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1799 AssertRC(rc);
1800}
1801
1802
1803/**
1804 * Flushes the guest TLB entry based on CPU capabilities.
1805 *
1806 * @param pVCpu Pointer to the VMCPU.
1807 * @param pCpu Pointer to the global HM CPU struct.
1808 */
1809DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1810{
1811#ifdef HMVMX_ALWAYS_FLUSH_TLB
1812 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1813#endif
1814 PVM pVM = pVCpu->CTX_SUFF(pVM);
1815 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1816 {
1817 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1818 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1819 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1820 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1821 default:
1822 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1823 break;
1824 }
1825 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH));
1826}
1827
1828
1829/**
1830 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1831 * TLB entries from the host TLB before VM-entry.
1832 *
1833 * @returns VBox status code.
1834 * @param pVM Pointer to the VM.
1835 */
1836static int hmR0VmxSetupTaggedTlb(PVM pVM)
1837{
1838 /*
1839 * Determine optimal flush type for Nested Paging.
1840 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1841 * guest execution (see hmR3InitFinalizeR0()).
1842 */
1843 if (pVM->hm.s.fNestedPaging)
1844 {
1845 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1846 {
1847 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1848 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1849 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1850 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1851 else
1852 {
1853 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1854 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1855 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1856 }
1857
1858 /* Make sure the write-back cacheable memory type for EPT is supported. */
1859 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1860 {
1861 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1862 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1863 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1864 }
1865 }
1866 else
1867 {
1868 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1869 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1870 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1871 }
1872 }
1873
1874 /*
1875 * Determine optimal flush type for VPID.
1876 */
1877 if (pVM->hm.s.vmx.fVpid)
1878 {
1879 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1880 {
1881 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1882 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1883 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1884 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1885 else
1886 {
1887 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1888 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1889 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1890 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1891 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1892 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1893 pVM->hm.s.vmx.fVpid = false;
1894 }
1895 }
1896 else
1897 {
1898 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1899 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1900 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1901 pVM->hm.s.vmx.fVpid = false;
1902 }
1903 }
1904
1905 /*
1906 * Setup the handler for flushing tagged-TLBs.
1907 */
1908 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1909 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1910 else if (pVM->hm.s.fNestedPaging)
1911 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1912 else if (pVM->hm.s.vmx.fVpid)
1913 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1914 else
1915 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1916 return VINF_SUCCESS;
1917}
1918
1919
1920/**
1921 * Sets up pin-based VM-execution controls in the VMCS.
1922 *
1923 * @returns VBox status code.
1924 * @param pVM Pointer to the VM.
1925 * @param pVCpu Pointer to the VMCPU.
1926 */
1927static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1928{
1929 AssertPtr(pVM);
1930 AssertPtr(pVCpu);
1931
1932 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1933 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1934
1935 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1936 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1937 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1938
1939 /* Enable the VMX preemption timer. */
1940 if (pVM->hm.s.vmx.fUsePreemptTimer)
1941 {
1942 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1943 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1944 }
1945
1946 if ((val & zap) != val)
1947 {
1948 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1949 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1950 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1951 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1952 }
1953
1954 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1955 AssertRCReturn(rc, rc);
1956
1957 /* Update VCPU with the currently set pin-based VM-execution controls. */
1958 pVCpu->hm.s.vmx.u32PinCtls = val;
1959 return rc;
1960}
1961
1962
1963/**
1964 * Sets up processor-based VM-execution controls in the VMCS.
1965 *
1966 * @returns VBox status code.
1967 * @param pVM Pointer to the VM.
1968 * @param pVMCPU Pointer to the VMCPU.
1969 */
1970static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1971{
1972 AssertPtr(pVM);
1973 AssertPtr(pVCpu);
1974
1975 int rc = VERR_INTERNAL_ERROR_5;
1976 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1977 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1978
1979 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1980 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1981 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1982 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1983 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1984 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1985 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1986
1987 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1988 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1989 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1990 {
1991 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1992 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1993 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1994 }
1995
1996 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1997 if (!pVM->hm.s.fNestedPaging)
1998 {
1999 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2000 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2001 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2002 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2003 }
2004
2005 /* Use TPR shadowing if supported by the CPU. */
2006 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2007 {
2008 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2009 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2010 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2011 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2012 AssertRCReturn(rc, rc);
2013
2014 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2015 /* CR8 writes causes a VM-exit based on TPR threshold. */
2016 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2017 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2018 }
2019 else
2020 {
2021 /*
2022 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2023 * Set this control only for 64-bit guests.
2024 */
2025 if (pVM->hm.s.fAllow64BitGuests)
2026 {
2027 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2028 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2029 }
2030 }
2031
2032 /* Use MSR-bitmaps if supported by the CPU. */
2033 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2034 {
2035 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2036
2037 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2038 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2039 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2040 AssertRCReturn(rc, rc);
2041
2042 /*
2043 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2044 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
2045 */
2046 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2047 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2048 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2049 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2050 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2051 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2052 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2053 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2054 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2055 }
2056
2057 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2058 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2059 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2060
2061 if ((val & zap) != val)
2062 {
2063 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2064 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2065 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2066 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2067 }
2068
2069 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2070 AssertRCReturn(rc, rc);
2071
2072 /* Update VCPU with the currently set processor-based VM-execution controls. */
2073 pVCpu->hm.s.vmx.u32ProcCtls = val;
2074
2075 /*
2076 * Secondary processor-based VM-execution controls.
2077 */
2078 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2079 {
2080 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2081 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2082
2083 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2084 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2085
2086 if (pVM->hm.s.fNestedPaging)
2087 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2088 else
2089 {
2090 /*
2091 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2092 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2093 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2094 */
2095 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2096 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2097 }
2098
2099 if (pVM->hm.s.vmx.fVpid)
2100 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2101
2102 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2103 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2104
2105 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2106 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2107 * done dynamically. */
2108 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2109 {
2110 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2111 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2112 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2113 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2114 AssertRCReturn(rc, rc);
2115 }
2116
2117 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2118 {
2119 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2120 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2121 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2122 }
2123
2124 if ((val & zap) != val)
2125 {
2126 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2127 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2128 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2129 }
2130
2131 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2132 AssertRCReturn(rc, rc);
2133
2134 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2135 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2136 }
2137 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2138 {
2139 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2140 "available\n"));
2141 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2142 }
2143
2144 return VINF_SUCCESS;
2145}
2146
2147
2148/**
2149 * Sets up miscellaneous (everything other than Pin & Processor-based
2150 * VM-execution) control fields in the VMCS.
2151 *
2152 * @returns VBox status code.
2153 * @param pVM Pointer to the VM.
2154 * @param pVCpu Pointer to the VMCPU.
2155 */
2156static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2157{
2158 AssertPtr(pVM);
2159 AssertPtr(pVCpu);
2160
2161 int rc = VERR_GENERAL_FAILURE;
2162
2163 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2164#if 0
2165 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2166 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2167 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2168
2169 /*
2170 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2171 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2172 * We thus use the exception bitmap to control it rather than use both.
2173 */
2174 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2175 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2176
2177 /** @todo Explore possibility of using IO-bitmaps. */
2178 /* All IO & IOIO instructions cause VM-exits. */
2179 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2180 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2181
2182 /* Initialize the MSR-bitmap area. */
2183 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2184 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2185 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2186#endif
2187
2188#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2189 /* Setup MSR autoloading/storing. */
2190 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2191 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2192 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2193 AssertRCReturn(rc, rc);
2194 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2195 AssertRCReturn(rc, rc);
2196
2197 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2198 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2199 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2200 AssertRCReturn(rc, rc);
2201#endif
2202
2203 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2204 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2205 AssertRCReturn(rc, rc);
2206
2207 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2208#if 0
2209 /* Setup debug controls */
2210 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2211 AssertRCReturn(rc, rc);
2212 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2213 AssertRCReturn(rc, rc);
2214#endif
2215
2216 return rc;
2217}
2218
2219
2220/**
2221 * Sets up the initial exception bitmap in the VMCS based on static conditions
2222 * (i.e. conditions that cannot ever change after starting the VM).
2223 *
2224 * @returns VBox status code.
2225 * @param pVM Pointer to the VM.
2226 * @param pVCpu Pointer to the VMCPU.
2227 */
2228static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2229{
2230 AssertPtr(pVM);
2231 AssertPtr(pVCpu);
2232
2233 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2234
2235 uint32_t u32XcptBitmap = 0;
2236
2237 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2238 if (!pVM->hm.s.fNestedPaging)
2239 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2240
2241 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2242 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2243 AssertRCReturn(rc, rc);
2244 return rc;
2245}
2246
2247
2248/**
2249 * Sets up the initial guest-state mask. The guest-state mask is consulted
2250 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2251 * for the nested virtualization case (as it would cause a VM-exit).
2252 *
2253 * @param pVCpu Pointer to the VMCPU.
2254 */
2255static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2256{
2257 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2258 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2259 return VINF_SUCCESS;
2260}
2261
2262
2263/**
2264 * Does per-VM VT-x initialization.
2265 *
2266 * @returns VBox status code.
2267 * @param pVM Pointer to the VM.
2268 */
2269VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2270{
2271 LogFlowFunc(("pVM=%p\n", pVM));
2272
2273 int rc = hmR0VmxStructsAlloc(pVM);
2274 if (RT_FAILURE(rc))
2275 {
2276 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2277 return rc;
2278 }
2279
2280 return VINF_SUCCESS;
2281}
2282
2283
2284/**
2285 * Does per-VM VT-x termination.
2286 *
2287 * @returns VBox status code.
2288 * @param pVM Pointer to the VM.
2289 */
2290VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2291{
2292 LogFlowFunc(("pVM=%p\n", pVM));
2293
2294#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2295 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2296 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2297#endif
2298 hmR0VmxStructsFree(pVM);
2299 return VINF_SUCCESS;
2300}
2301
2302
2303/**
2304 * Sets up the VM for execution under VT-x.
2305 * This function is only called once per-VM during initialization.
2306 *
2307 * @returns VBox status code.
2308 * @param pVM Pointer to the VM.
2309 */
2310VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2311{
2312 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2313 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2314
2315 LogFlowFunc(("pVM=%p\n", pVM));
2316
2317 /*
2318 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2319 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2320 */
2321 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2322 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2323 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2324 || !pVM->hm.s.vmx.pRealModeTSS))
2325 {
2326 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2327 return VERR_INTERNAL_ERROR;
2328 }
2329
2330#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2331 /*
2332 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2333 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2334 */
2335 if ( pVM->hm.s.fAllow64BitGuests
2336 && !HMVMX_IS_64BIT_HOST_MODE())
2337 {
2338 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2339 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2340 }
2341#endif
2342
2343 /* Initialize these always, see hmR3InitFinalizeR0().*/
2344 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2345 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2346
2347 /* Setup the tagged-TLB flush handlers. */
2348 int rc = hmR0VmxSetupTaggedTlb(pVM);
2349 if (RT_FAILURE(rc))
2350 {
2351 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2352 return rc;
2353 }
2354
2355 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2356 {
2357 PVMCPU pVCpu = &pVM->aCpus[i];
2358 AssertPtr(pVCpu);
2359 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2360
2361 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2362 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2363
2364 /* Set revision dword at the beginning of the VMCS structure. */
2365 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2366
2367 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2368 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2369 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2370 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2371
2372 /* Load this VMCS as the current VMCS. */
2373 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2374 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2375 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2376
2377 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2378 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2379 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2380
2381 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2382 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2383 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2384
2385 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2386 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2387 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2388
2389 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2390 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2391 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2392
2393 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2394 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2395 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2396
2397#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2398 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2399 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2400 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2401#endif
2402
2403 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2404 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2405 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2406 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2407
2408 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2409
2410 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2411 }
2412
2413 return VINF_SUCCESS;
2414}
2415
2416
2417/**
2418 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2419 * the VMCS.
2420 *
2421 * @returns VBox status code.
2422 * @param pVM Pointer to the VM.
2423 * @param pVCpu Pointer to the VMCPU.
2424 */
2425DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2426{
2427 RTCCUINTREG uReg = ASMGetCR0();
2428 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2429 AssertRCReturn(rc, rc);
2430
2431#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2432 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2433 if (HMVMX_IS_64BIT_HOST_MODE())
2434 {
2435 uint64_t uRegCR3 = HMR0Get64bitCR3();
2436 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2437 }
2438 else
2439#endif
2440 {
2441 uReg = ASMGetCR3();
2442 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2443 }
2444 AssertRCReturn(rc, rc);
2445
2446 uReg = ASMGetCR4();
2447 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2448 AssertRCReturn(rc, rc);
2449 return rc;
2450}
2451
2452
2453#if HC_ARCH_BITS == 64
2454/**
2455 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2456 * requirements. See hmR0VmxSaveHostSegmentRegs().
2457 */
2458# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2459 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2460 { \
2461 bool fValidSelector = true; \
2462 if ((selValue) & X86_SEL_LDT) \
2463 { \
2464 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2465 fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2466 } \
2467 if (fValidSelector) \
2468 { \
2469 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2470 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2471 } \
2472 (selValue) = 0; \
2473 }
2474#endif
2475
2476
2477/**
2478 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2479 * the host-state area in the VMCS.
2480 *
2481 * @returns VBox status code.
2482 * @param pVM Pointer to the VM.
2483 * @param pVCpu Pointer to the VMCPU.
2484 */
2485DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2486{
2487 int rc = VERR_INTERNAL_ERROR_5;
2488
2489 /*
2490 * Host DS, ES, FS and GS segment registers.
2491 */
2492#if HC_ARCH_BITS == 64
2493 RTSEL uSelDS = ASMGetDS();
2494 RTSEL uSelES = ASMGetES();
2495 RTSEL uSelFS = ASMGetFS();
2496 RTSEL uSelGS = ASMGetGS();
2497#else
2498 RTSEL uSelDS = 0;
2499 RTSEL uSelES = 0;
2500 RTSEL uSelFS = 0;
2501 RTSEL uSelGS = 0;
2502#endif
2503
2504 /* Recalculate which host-state bits need to be manually restored. */
2505 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2506
2507 /*
2508 * Host CS and SS segment registers.
2509 */
2510#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2511 RTSEL uSelCS;
2512 RTSEL uSelSS;
2513 if (HMVMX_IS_64BIT_HOST_MODE())
2514 {
2515 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2516 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2517 }
2518 else
2519 {
2520 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2521 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2522 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2523 }
2524#else
2525 RTSEL uSelCS = ASMGetCS();
2526 RTSEL uSelSS = ASMGetSS();
2527#endif
2528
2529 /*
2530 * Host TR segment register.
2531 */
2532 RTSEL uSelTR = ASMGetTR();
2533
2534#if HC_ARCH_BITS == 64
2535 /*
2536 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2537 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2538 */
2539 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2540 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2541 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2542 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2543# undef VMXLOCAL_ADJUST_HOST_SEG
2544#endif
2545
2546 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2547 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2548 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2549 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2550 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2551 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2552 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2553 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2554 Assert(uSelCS);
2555 Assert(uSelTR);
2556
2557 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2558#if 0
2559 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2560 Assert(uSelSS != 0);
2561#endif
2562
2563 /* Write these host selector fields into the host-state area in the VMCS. */
2564 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2565 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2566#if HC_ARCH_BITS == 64
2567 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2568 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2570 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2571#endif
2572 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2573
2574 /*
2575 * Host GDTR and IDTR.
2576 */
2577 RTGDTR Gdtr;
2578 RT_ZERO(Gdtr);
2579#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2580 if (HMVMX_IS_64BIT_HOST_MODE())
2581 {
2582 X86XDTR64 Gdtr64;
2583 X86XDTR64 Idtr64;
2584 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2585 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2586 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2587
2588 Gdtr.cbGdt = Gdtr64.cb;
2589 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2590 }
2591 else
2592#endif
2593 {
2594 RTIDTR Idtr;
2595 ASMGetGDTR(&Gdtr);
2596 ASMGetIDTR(&Idtr);
2597 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2598 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2599
2600#if HC_ARCH_BITS == 64
2601 /*
2602 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2603 * maximum limit (0xffff) on every VM-exit.
2604 */
2605 if (Gdtr.cbGdt != 0xffff)
2606 {
2607 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2608 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2609 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2610 }
2611
2612 /*
2613 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2614 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2615 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2616 */
2617 if (Idtr.cbIdt < 0x0fff)
2618 {
2619 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2620 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2621 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2622 }
2623#endif
2624 }
2625
2626 /*
2627 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2628 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2629 */
2630 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2631 {
2632 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2633 return VERR_VMX_INVALID_HOST_STATE;
2634 }
2635
2636 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2637#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2638 if (HMVMX_IS_64BIT_HOST_MODE())
2639 {
2640 /* We need the 64-bit TR base for hybrid darwin. */
2641 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2642 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2643 }
2644 else
2645#endif
2646 {
2647 uintptr_t uTRBase;
2648#if HC_ARCH_BITS == 64
2649 uTRBase = X86DESC64_BASE(pDesc);
2650
2651 /*
2652 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2653 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2654 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2655 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2656 *
2657 * [1] See Intel spec. 3.5 "System Descriptor Types".
2658 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2659 */
2660 Assert(pDesc->System.u4Type == 11);
2661 if ( pDesc->System.u16LimitLow != 0x67
2662 || pDesc->System.u4LimitHigh)
2663 {
2664 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2665 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2666
2667 /* Store the GDTR here as we need it while restoring TR. */
2668 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2669 }
2670#else
2671 uTRBase = X86DESC_BASE(pDesc);
2672#endif
2673 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2674 }
2675 AssertRCReturn(rc, rc);
2676
2677 /*
2678 * Host FS base and GS base.
2679 */
2680#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2681 if (HMVMX_IS_64BIT_HOST_MODE())
2682 {
2683 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2684 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2685 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2686 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2687
2688# if HC_ARCH_BITS == 64
2689 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2690 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2691 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2692 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2693 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2694# endif
2695 }
2696#endif
2697 return rc;
2698}
2699
2700
2701/**
2702 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2703 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2704 * the host after every successful VM exit.
2705 *
2706 * @returns VBox status code.
2707 * @param pVM Pointer to the VM.
2708 * @param pVCpu Pointer to the VMCPU.
2709 */
2710DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2711{
2712 AssertPtr(pVCpu);
2713 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2714
2715 int rc = VINF_SUCCESS;
2716#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2718 uint32_t cHostMsrs = 0;
2719 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2720
2721 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2722 {
2723 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2724
2725# if HC_ARCH_BITS == 64
2726 /* Paranoia. 64-bit code requires these bits to be set always. */
2727 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2728
2729 /*
2730 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2731 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2732 * some reason (e.g. allow transparent reads) we would activate the code below.
2733 */
2734# if 0
2735 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2736 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2737 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2738 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2739 if (CPUMIsGuestInLongMode(pVCpu))
2740 {
2741 uint64_t u64GuestEfer;
2742 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2743 AssertRC(rc);
2744
2745 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2746 {
2747 pHostMsr->u32Msr = MSR_K6_EFER;
2748 pHostMsr->u32Reserved = 0;
2749 pHostMsr->u64Value = u64HostEfer;
2750 pHostMsr++; cHostMsrs++;
2751 }
2752 }
2753# endif
2754# else /* HC_ARCH_BITS != 64 */
2755 pHostMsr->u32Msr = MSR_K6_EFER;
2756 pHostMsr->u32Reserved = 0;
2757# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2758 if (CPUMIsGuestInLongMode(pVCpu))
2759 {
2760 /* Must match the EFER value in our 64 bits switcher. */
2761 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2762 }
2763 else
2764# endif
2765 pHostMsr->u64Value = u64HostEfer;
2766 pHostMsr++; cHostMsrs++;
2767# endif /* HC_ARCH_BITS == 64 */
2768 }
2769
2770# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2771 if (HMVMX_IS_64BIT_HOST_MODE())
2772 {
2773 pHostMsr->u32Msr = MSR_K6_STAR;
2774 pHostMsr->u32Reserved = 0;
2775 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2776 pHostMsr++; cHostMsrs++;
2777 pHostMsr->u32Msr = MSR_K8_LSTAR;
2778 pHostMsr->u32Reserved = 0;
2779 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2780 pHostMsr++; cHostMsrs++;
2781 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2782 pHostMsr->u32Reserved = 0;
2783 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2784 pHostMsr++; cHostMsrs++;
2785 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2786 pHostMsr->u32Reserved = 0;
2787 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2788 pHostMsr++; cHostMsrs++;
2789 }
2790# endif
2791
2792 /* Host TSC AUX MSR must be restored since we always load/store guest TSC AUX MSR. */
2793 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2794 {
2795 pHostMsr->u32Msr = MSR_K8_TSC_AUX;
2796 pHostMsr->u32Reserved = 0;
2797 pHostMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
2798 pHostMsr++; cHostMsrs++;
2799 }
2800
2801 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2802 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2803 {
2804 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2805 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2806 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2807 }
2808
2809 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2810#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2811
2812 /*
2813 * Host Sysenter MSRs.
2814 */
2815 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2816 AssertRCReturn(rc, rc);
2817#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2818 if (HMVMX_IS_64BIT_HOST_MODE())
2819 {
2820 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2821 AssertRCReturn(rc, rc);
2822 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2823 }
2824 else
2825 {
2826 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2827 AssertRCReturn(rc, rc);
2828 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2829 }
2830#elif HC_ARCH_BITS == 32
2831 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2832 AssertRCReturn(rc, rc);
2833 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2834#else
2835 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2836 AssertRCReturn(rc, rc);
2837 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2838#endif
2839 AssertRCReturn(rc, rc);
2840
2841 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2842 * hmR0VmxSetupExitCtls() !! */
2843 return rc;
2844}
2845
2846
2847/**
2848 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2849 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2850 * controls".
2851 *
2852 * @returns VBox status code.
2853 * @param pVCpu Pointer to the VMCPU.
2854 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2855 * out-of-sync. Make sure to update the required fields
2856 * before using them.
2857 *
2858 * @remarks No-long-jump zone!!!
2859 */
2860DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2861{
2862 int rc = VINF_SUCCESS;
2863 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2864 {
2865 PVM pVM = pVCpu->CTX_SUFF(pVM);
2866 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2867 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2868
2869 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2870 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2871
2872 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2873 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2874 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2875 else
2876 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2877
2878 /*
2879 * The following should -not- be set (since we're not in SMM mode):
2880 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2881 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2882 */
2883
2884 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2885 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2886 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2887
2888 if ((val & zap) != val)
2889 {
2890 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2891 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2892 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2893 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2894 }
2895
2896 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2897 AssertRCReturn(rc, rc);
2898
2899 /* Update VCPU with the currently set VM-exit controls. */
2900 pVCpu->hm.s.vmx.u32EntryCtls = val;
2901 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2902 }
2903 return rc;
2904}
2905
2906
2907/**
2908 * Sets up the VM-exit controls in the VMCS.
2909 *
2910 * @returns VBox status code.
2911 * @param pVM Pointer to the VM.
2912 * @param pVCpu Pointer to the VMCPU.
2913 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2914 * out-of-sync. Make sure to update the required fields
2915 * before using them.
2916 *
2917 * @remarks requires EFER.
2918 */
2919DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2920{
2921 int rc = VINF_SUCCESS;
2922 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2923 {
2924 PVM pVM = pVCpu->CTX_SUFF(pVM);
2925 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2926 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2927
2928 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2929 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2930
2931 /*
2932 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2933 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2934 */
2935#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2936 if (HMVMX_IS_64BIT_HOST_MODE())
2937 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2938 else
2939 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2940#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2941 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2942 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2943 else
2944 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2945#endif
2946
2947 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2948 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2949
2950 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2951 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2952 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2953 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2954 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2955
2956 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2957 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2958
2959 if ((val & zap) != val)
2960 {
2961 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2962 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2963 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2964 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2965 }
2966
2967 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2968 AssertRCReturn(rc, rc);
2969
2970 /* Update VCPU with the currently set VM-exit controls. */
2971 pVCpu->hm.s.vmx.u32ExitCtls = val;
2972 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2973 }
2974 return rc;
2975}
2976
2977
2978/**
2979 * Loads the guest APIC and related state.
2980 *
2981 * @returns VBox status code.
2982 * @param pVM Pointer to the VM.
2983 * @param pVCpu Pointer to the VMCPU.
2984 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2985 * out-of-sync. Make sure to update the required fields
2986 * before using them.
2987 */
2988DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2989{
2990 int rc = VINF_SUCCESS;
2991 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2992 {
2993 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2994 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2995 {
2996 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2997
2998 bool fPendingIntr = false;
2999 uint8_t u8Tpr = 0;
3000 uint8_t u8PendingIntr = 0;
3001 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3002 AssertRCReturn(rc, rc);
3003
3004 /*
3005 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3006 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3007 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3008 * the interrupt when we VM-exit for other reasons.
3009 */
3010 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3011 uint32_t u32TprThreshold = 0;
3012 if (fPendingIntr)
3013 {
3014 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
3015 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
3016 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
3017 if (u8PendingPriority <= u8TprPriority)
3018 u32TprThreshold = u8PendingPriority;
3019 else
3020 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3021 }
3022 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3023
3024 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3025 AssertRCReturn(rc, rc);
3026 }
3027
3028 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3029 }
3030 return rc;
3031}
3032
3033
3034/**
3035 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3036 *
3037 * @returns Guest's interruptibility-state.
3038 * @param pVCpu Pointer to the VMCPU.
3039 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3040 * out-of-sync. Make sure to update the required fields
3041 * before using them.
3042 *
3043 * @remarks No-long-jump zone!!!
3044 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3045 */
3046DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3047{
3048 /*
3049 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3050 * inhibit interrupts or clear any existing interrupt-inhibition.
3051 */
3052 uint32_t uIntrState = 0;
3053 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3054 {
3055 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3056 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
3057 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3058 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3059 {
3060 /*
3061 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3062 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3063 */
3064 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3065 }
3066 else if (pMixedCtx->eflags.Bits.u1IF)
3067 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3068 else
3069 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3070 }
3071 return uIntrState;
3072}
3073
3074
3075/**
3076 * Loads the guest's interruptibility-state into the guest-state area in the
3077 * VMCS.
3078 *
3079 * @returns VBox status code.
3080 * @param pVCpu Pointer to the VMCPU.
3081 * @param uIntrState The interruptibility-state to set.
3082 */
3083static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3084{
3085 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3086 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3087 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3088 AssertRCReturn(rc, rc);
3089 return rc;
3090}
3091
3092
3093/**
3094 * Loads the guest's RIP into the guest-state area in the VMCS.
3095 *
3096 * @returns VBox status code.
3097 * @param pVCpu Pointer to the VMCPU.
3098 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3099 * out-of-sync. Make sure to update the required fields
3100 * before using them.
3101 *
3102 * @remarks No-long-jump zone!!!
3103 */
3104static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3105{
3106 int rc = VINF_SUCCESS;
3107 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3108 {
3109 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3110 AssertRCReturn(rc, rc);
3111
3112 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3113 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3114 }
3115 return rc;
3116}
3117
3118
3119/**
3120 * Loads the guest's RSP into the guest-state area in the VMCS.
3121 *
3122 * @returns VBox status code.
3123 * @param pVCpu Pointer to the VMCPU.
3124 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3125 * out-of-sync. Make sure to update the required fields
3126 * before using them.
3127 *
3128 * @remarks No-long-jump zone!!!
3129 */
3130static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3131{
3132 int rc = VINF_SUCCESS;
3133 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3134 {
3135 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3136 AssertRCReturn(rc, rc);
3137
3138 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3139 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3140 }
3141 return rc;
3142}
3143
3144
3145/**
3146 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3147 *
3148 * @returns VBox status code.
3149 * @param pVCpu Pointer to the VMCPU.
3150 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3151 * out-of-sync. Make sure to update the required fields
3152 * before using them.
3153 *
3154 * @remarks No-long-jump zone!!!
3155 */
3156static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3157{
3158 int rc = VINF_SUCCESS;
3159 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3160 {
3161 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3162 Let us assert it as such and use 32-bit VMWRITE. */
3163 Assert(!(pMixedCtx->rflags.u64 >> 32));
3164 X86EFLAGS Eflags = pMixedCtx->eflags;
3165 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3166 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3167
3168 /*
3169 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3170 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3171 */
3172 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3173 {
3174 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3175 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3176 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3177 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3178 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3179 }
3180
3181 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3182 AssertRCReturn(rc, rc);
3183
3184 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3185 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3186 }
3187 return rc;
3188}
3189
3190
3191/**
3192 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3193 *
3194 * @returns VBox status code.
3195 * @param pVCpu Pointer to the VMCPU.
3196 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3197 * out-of-sync. Make sure to update the required fields
3198 * before using them.
3199 *
3200 * @remarks No-long-jump zone!!!
3201 */
3202DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3203{
3204 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3205 AssertRCReturn(rc, rc);
3206 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3207 AssertRCReturn(rc, rc);
3208 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3209 AssertRCReturn(rc, rc);
3210 return rc;
3211}
3212
3213
3214/**
3215 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3216 * CR0 is partially shared with the host and we have to consider the FPU bits.
3217 *
3218 * @returns VBox status code.
3219 * @param pVM Pointer to the VM.
3220 * @param pVCpu Pointer to the VMCPU.
3221 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3222 * out-of-sync. Make sure to update the required fields
3223 * before using them.
3224 *
3225 * @remarks No-long-jump zone!!!
3226 */
3227static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3228{
3229 /*
3230 * Guest CR0.
3231 * Guest FPU.
3232 */
3233 int rc = VINF_SUCCESS;
3234 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3235 {
3236 Assert(!(pMixedCtx->cr0 >> 32));
3237 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3238 PVM pVM = pVCpu->CTX_SUFF(pVM);
3239
3240 /* The guest's view (read access) of its CR0 is unblemished. */
3241 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3242 AssertRCReturn(rc, rc);
3243 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3244
3245 /* Setup VT-x's view of the guest CR0. */
3246 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3247 if (pVM->hm.s.fNestedPaging)
3248 {
3249 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3250 {
3251 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3252 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3253 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3254 }
3255 else
3256 {
3257 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3258 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3259 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3260 }
3261
3262 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3263 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3264 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3265
3266 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3267 AssertRCReturn(rc, rc);
3268 }
3269 else
3270 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3271
3272 /*
3273 * Guest FPU bits.
3274 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3275 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3276 */
3277 u32GuestCR0 |= X86_CR0_NE;
3278 bool fInterceptNM = false;
3279 if (CPUMIsGuestFPUStateActive(pVCpu))
3280 {
3281 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3282 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3283 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3284 }
3285 else
3286 {
3287 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3288 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3289 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3290 }
3291
3292 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3293 bool fInterceptMF = false;
3294 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3295 fInterceptMF = true;
3296
3297 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3298 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3299 {
3300 Assert(PDMVmmDevHeapIsEnabled(pVM));
3301 Assert(pVM->hm.s.vmx.pRealModeTSS);
3302 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3303 fInterceptNM = true;
3304 fInterceptMF = true;
3305 }
3306 else
3307 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3308
3309 if (fInterceptNM)
3310 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3311 else
3312 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3313
3314 if (fInterceptMF)
3315 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3316 else
3317 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3318
3319 /* Additional intercepts for debugging, define these yourself explicitly. */
3320#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3321 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3322 | RT_BIT(X86_XCPT_BP)
3323 | RT_BIT(X86_XCPT_DB)
3324 | RT_BIT(X86_XCPT_DE)
3325 | RT_BIT(X86_XCPT_NM)
3326 | RT_BIT(X86_XCPT_UD)
3327 | RT_BIT(X86_XCPT_NP)
3328 | RT_BIT(X86_XCPT_SS)
3329 | RT_BIT(X86_XCPT_GP)
3330 | RT_BIT(X86_XCPT_PF)
3331 | RT_BIT(X86_XCPT_MF)
3332 ;
3333#elif defined(HMVMX_ALWAYS_TRAP_PF)
3334 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3335#endif
3336
3337 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3338
3339 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3340 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3341 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3342 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3343 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3344 else
3345 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3346
3347 u32GuestCR0 |= uSetCR0;
3348 u32GuestCR0 &= uZapCR0;
3349 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3350
3351 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3352 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3353 AssertRCReturn(rc, rc);
3354 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3355 AssertRCReturn(rc, rc);
3356 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3357
3358 /*
3359 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3360 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3361 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3362 */
3363 uint32_t u32CR0Mask = 0;
3364 u32CR0Mask = X86_CR0_PE
3365 | X86_CR0_NE
3366 | X86_CR0_WP
3367 | X86_CR0_PG
3368 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3369 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3370 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3371
3372 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3373 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3374 * and @bugref{6944}. */
3375#if 0
3376 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3377 u32CR0Mask &= ~X86_CR0_PE;
3378#endif
3379 if (pVM->hm.s.fNestedPaging)
3380 u32CR0Mask &= ~X86_CR0_WP;
3381
3382 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3383 if (fInterceptNM)
3384 {
3385 u32CR0Mask |= X86_CR0_TS
3386 | X86_CR0_MP;
3387 }
3388
3389 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3390 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3391 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3392 AssertRCReturn(rc, rc);
3393 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3394
3395 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3396 }
3397 return rc;
3398}
3399
3400
3401/**
3402 * Loads the guest control registers (CR3, CR4) into the guest-state area
3403 * in the VMCS.
3404 *
3405 * @returns VBox status code.
3406 * @param pVM Pointer to the VM.
3407 * @param pVCpu Pointer to the VMCPU.
3408 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3409 * out-of-sync. Make sure to update the required fields
3410 * before using them.
3411 *
3412 * @remarks No-long-jump zone!!!
3413 */
3414static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3415{
3416 int rc = VINF_SUCCESS;
3417 PVM pVM = pVCpu->CTX_SUFF(pVM);
3418
3419 /*
3420 * Guest CR2.
3421 * It's always loaded in the assembler code. Nothing to do here.
3422 */
3423
3424 /*
3425 * Guest CR3.
3426 */
3427 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3428 {
3429 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3430 if (pVM->hm.s.fNestedPaging)
3431 {
3432 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3433
3434 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3435 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3436 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3437 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3438
3439 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3440 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3441 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3442
3443 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3444 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3445 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3446 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3447
3448 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3449 AssertRCReturn(rc, rc);
3450 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3451
3452 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3453 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3454 {
3455 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3456 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3457 {
3458 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3459 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3460 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3461 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3462 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3463 }
3464
3465 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3466 have Unrestricted Execution to handle the guest when it's not using paging. */
3467 GCPhysGuestCR3 = pMixedCtx->cr3;
3468 }
3469 else
3470 {
3471 /*
3472 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3473 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3474 * EPT takes care of translating it to host-physical addresses.
3475 */
3476 RTGCPHYS GCPhys;
3477 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3478 Assert(PDMVmmDevHeapIsEnabled(pVM));
3479
3480 /* We obtain it here every time as the guest could have relocated this PCI region. */
3481 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3482 AssertRCReturn(rc, rc);
3483
3484 GCPhysGuestCR3 = GCPhys;
3485 }
3486
3487 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3488 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3489 }
3490 else
3491 {
3492 /* Non-nested paging case, just use the hypervisor's CR3. */
3493 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3494
3495 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3496 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3497 }
3498 AssertRCReturn(rc, rc);
3499
3500 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3501 }
3502
3503 /*
3504 * Guest CR4.
3505 */
3506 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3507 {
3508 Assert(!(pMixedCtx->cr4 >> 32));
3509 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3510
3511 /* The guest's view of its CR4 is unblemished. */
3512 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3513 AssertRCReturn(rc, rc);
3514 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3515
3516 /* Setup VT-x's view of the guest CR4. */
3517 /*
3518 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3519 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3520 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3521 */
3522 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3523 {
3524 Assert(pVM->hm.s.vmx.pRealModeTSS);
3525 Assert(PDMVmmDevHeapIsEnabled(pVM));
3526 u32GuestCR4 &= ~X86_CR4_VME;
3527 }
3528
3529 if (pVM->hm.s.fNestedPaging)
3530 {
3531 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3532 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3533 {
3534 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3535 u32GuestCR4 |= X86_CR4_PSE;
3536 /* Our identity mapping is a 32 bits page directory. */
3537 u32GuestCR4 &= ~X86_CR4_PAE;
3538 }
3539 /* else use guest CR4.*/
3540 }
3541 else
3542 {
3543 /*
3544 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3545 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3546 */
3547 switch (pVCpu->hm.s.enmShadowMode)
3548 {
3549 case PGMMODE_REAL: /* Real-mode. */
3550 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3551 case PGMMODE_32_BIT: /* 32-bit paging. */
3552 {
3553 u32GuestCR4 &= ~X86_CR4_PAE;
3554 break;
3555 }
3556
3557 case PGMMODE_PAE: /* PAE paging. */
3558 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3559 {
3560 u32GuestCR4 |= X86_CR4_PAE;
3561 break;
3562 }
3563
3564 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3565 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3566#ifdef VBOX_ENABLE_64_BITS_GUESTS
3567 break;
3568#endif
3569 default:
3570 AssertFailed();
3571 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3572 }
3573 }
3574
3575 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3576 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3577 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3578 u32GuestCR4 |= uSetCR4;
3579 u32GuestCR4 &= uZapCR4;
3580
3581 /* Write VT-x's view of the guest CR4 into the VMCS. */
3582 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3583 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3584 AssertRCReturn(rc, rc);
3585
3586 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3587 uint32_t u32CR4Mask = 0;
3588 u32CR4Mask = X86_CR4_VME
3589 | X86_CR4_PAE
3590 | X86_CR4_PGE
3591 | X86_CR4_PSE
3592 | X86_CR4_VMXE;
3593 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3594 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3595 AssertRCReturn(rc, rc);
3596
3597 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3598 }
3599 return rc;
3600}
3601
3602
3603/**
3604 * Loads the guest debug registers into the guest-state area in the VMCS.
3605 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3606 *
3607 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu Pointer to the VMCPU.
3611 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3612 * out-of-sync. Make sure to update the required fields
3613 * before using them.
3614 *
3615 * @remarks No-long-jump zone!!!
3616 */
3617static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3618{
3619 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3620 return VINF_SUCCESS;
3621
3622#ifdef VBOX_STRICT
3623 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3624 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3625 {
3626 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3627 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3628 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3629 }
3630#endif
3631
3632 int rc;
3633 PVM pVM = pVCpu->CTX_SUFF(pVM);
3634 bool fInterceptDB = false;
3635 bool fInterceptMovDRx = false;
3636 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3637 {
3638 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3639 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3640 {
3641 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3642 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3643 AssertRCReturn(rc, rc);
3644 Assert(fInterceptDB == false);
3645 }
3646 else
3647 {
3648 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3649 pVCpu->hm.s.fClearTrapFlag = true;
3650 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3651 fInterceptDB = true;
3652 }
3653 }
3654
3655 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3656 {
3657 /*
3658 * Use the combined guest and host DRx values found in the hypervisor
3659 * register set because the debugger has breakpoints active or someone
3660 * is single stepping on the host side without a monitor trap flag.
3661 *
3662 * Note! DBGF expects a clean DR6 state before executing guest code.
3663 */
3664#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3665 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3666 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3667 {
3668 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3669 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3670 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3671 }
3672 else
3673#endif
3674 if (!CPUMIsHyperDebugStateActive(pVCpu))
3675 {
3676 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3677 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3678 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3679 }
3680
3681 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3682 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3683 AssertRCReturn(rc, rc);
3684
3685 pVCpu->hm.s.fUsingHyperDR7 = true;
3686 fInterceptDB = true;
3687 fInterceptMovDRx = true;
3688 }
3689 else
3690 {
3691 /*
3692 * If the guest has enabled debug registers, we need to load them prior to
3693 * executing guest code so they'll trigger at the right time.
3694 */
3695 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3696 {
3697#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3698 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3699 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3700 {
3701 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3702 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3703 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3704 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3705 }
3706 else
3707#endif
3708 if (CPUMIsGuestDebugStateActive(pVCpu))
3709 {
3710 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3711 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3712 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3713 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3714 }
3715 }
3716 /*
3717 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3718 * must intercept #DB in order to maintain a correct DR6 guest value.
3719 */
3720#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3721 else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3722 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3723 || !CPUMIsGuestDebugStateActive(pVCpu))
3724#else
3725 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3726#endif
3727 {
3728 fInterceptMovDRx = true;
3729 fInterceptDB = true;
3730 }
3731
3732 /* Update guest DR7. */
3733 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3734 AssertRCReturn(rc, rc);
3735
3736 pVCpu->hm.s.fUsingHyperDR7 = false;
3737 }
3738
3739 /*
3740 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3741 */
3742 if (fInterceptDB)
3743 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3744 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3745 {
3746#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3747 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3748#endif
3749 }
3750 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3751 AssertRCReturn(rc, rc);
3752
3753 /*
3754 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3755 */
3756 if (fInterceptMovDRx)
3757 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3758 else
3759 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3760 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3761 AssertRCReturn(rc, rc);
3762
3763 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3764 return VINF_SUCCESS;
3765}
3766
3767
3768#ifdef VBOX_STRICT
3769/**
3770 * Strict function to validate segment registers.
3771 *
3772 * @remarks ASSUMES CR0 is up to date.
3773 */
3774static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3775{
3776 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3777 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3778 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3779 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3780 && ( !CPUMIsGuestInRealModeEx(pCtx)
3781 && !CPUMIsGuestInV86ModeEx(pCtx)))
3782 {
3783 /* Protected mode checks */
3784 /* CS */
3785 Assert(pCtx->cs.Attr.n.u1Present);
3786 Assert(!(pCtx->cs.Attr.u & 0xf00));
3787 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3788 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3789 || !(pCtx->cs.Attr.n.u1Granularity));
3790 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3791 || (pCtx->cs.Attr.n.u1Granularity));
3792 /* CS cannot be loaded with NULL in protected mode. */
3793 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3794 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3795 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3796 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3797 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3798 else
3799 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3800 /* SS */
3801 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3802 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3803 if ( !(pCtx->cr0 & X86_CR0_PE)
3804 || pCtx->cs.Attr.n.u4Type == 3)
3805 {
3806 Assert(!pCtx->ss.Attr.n.u2Dpl);
3807 }
3808 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3809 {
3810 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3811 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3812 Assert(pCtx->ss.Attr.n.u1Present);
3813 Assert(!(pCtx->ss.Attr.u & 0xf00));
3814 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3815 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3816 || !(pCtx->ss.Attr.n.u1Granularity));
3817 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3818 || (pCtx->ss.Attr.n.u1Granularity));
3819 }
3820 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3821 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3822 {
3823 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3824 Assert(pCtx->ds.Attr.n.u1Present);
3825 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3826 Assert(!(pCtx->ds.Attr.u & 0xf00));
3827 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3828 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3829 || !(pCtx->ds.Attr.n.u1Granularity));
3830 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3831 || (pCtx->ds.Attr.n.u1Granularity));
3832 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3833 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3834 }
3835 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3836 {
3837 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3838 Assert(pCtx->es.Attr.n.u1Present);
3839 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3840 Assert(!(pCtx->es.Attr.u & 0xf00));
3841 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3842 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3843 || !(pCtx->es.Attr.n.u1Granularity));
3844 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3845 || (pCtx->es.Attr.n.u1Granularity));
3846 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3847 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3848 }
3849 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3850 {
3851 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3852 Assert(pCtx->fs.Attr.n.u1Present);
3853 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3854 Assert(!(pCtx->fs.Attr.u & 0xf00));
3855 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3856 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3857 || !(pCtx->fs.Attr.n.u1Granularity));
3858 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3859 || (pCtx->fs.Attr.n.u1Granularity));
3860 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3861 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3862 }
3863 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3864 {
3865 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3866 Assert(pCtx->gs.Attr.n.u1Present);
3867 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3868 Assert(!(pCtx->gs.Attr.u & 0xf00));
3869 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3870 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3871 || !(pCtx->gs.Attr.n.u1Granularity));
3872 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3873 || (pCtx->gs.Attr.n.u1Granularity));
3874 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3875 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3876 }
3877 /* 64-bit capable CPUs. */
3878# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3879 Assert(!(pCtx->cs.u64Base >> 32));
3880 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3881 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3882 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3883# endif
3884 }
3885 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3886 || ( CPUMIsGuestInRealModeEx(pCtx)
3887 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3888 {
3889 /* Real and v86 mode checks. */
3890 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3891 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3892 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3893 {
3894 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3895 }
3896 else
3897 {
3898 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3899 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3900 }
3901
3902 /* CS */
3903 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3904 Assert(pCtx->cs.u32Limit == 0xffff);
3905 Assert(u32CSAttr == 0xf3);
3906 /* SS */
3907 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3908 Assert(pCtx->ss.u32Limit == 0xffff);
3909 Assert(u32SSAttr == 0xf3);
3910 /* DS */
3911 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3912 Assert(pCtx->ds.u32Limit == 0xffff);
3913 Assert(u32DSAttr == 0xf3);
3914 /* ES */
3915 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3916 Assert(pCtx->es.u32Limit == 0xffff);
3917 Assert(u32ESAttr == 0xf3);
3918 /* FS */
3919 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3920 Assert(pCtx->fs.u32Limit == 0xffff);
3921 Assert(u32FSAttr == 0xf3);
3922 /* GS */
3923 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3924 Assert(pCtx->gs.u32Limit == 0xffff);
3925 Assert(u32GSAttr == 0xf3);
3926 /* 64-bit capable CPUs. */
3927# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3928 Assert(!(pCtx->cs.u64Base >> 32));
3929 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3930 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3931 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3932# endif
3933 }
3934}
3935#endif /* VBOX_STRICT */
3936
3937
3938/**
3939 * Writes a guest segment register into the guest-state area in the VMCS.
3940 *
3941 * @returns VBox status code.
3942 * @param pVCpu Pointer to the VMCPU.
3943 * @param idxSel Index of the selector in the VMCS.
3944 * @param idxLimit Index of the segment limit in the VMCS.
3945 * @param idxBase Index of the segment base in the VMCS.
3946 * @param idxAccess Index of the access rights of the segment in the VMCS.
3947 * @param pSelReg Pointer to the segment selector.
3948 * @param pCtx Pointer to the guest-CPU context.
3949 *
3950 * @remarks No-long-jump zone!!!
3951 */
3952static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3953 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3954{
3955 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3956 AssertRCReturn(rc, rc);
3957 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3958 AssertRCReturn(rc, rc);
3959 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3960 AssertRCReturn(rc, rc);
3961
3962 uint32_t u32Access = pSelReg->Attr.u;
3963 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3964 {
3965 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3966 u32Access = 0xf3;
3967 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3968 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3969 }
3970 else
3971 {
3972 /*
3973 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3974 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3975 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3976 * loaded in protected-mode have their attribute as 0.
3977 */
3978 if (!u32Access)
3979 u32Access = X86DESCATTR_UNUSABLE;
3980 }
3981
3982 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3983 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3984 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3985
3986 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3987 AssertRCReturn(rc, rc);
3988 return rc;
3989}
3990
3991
3992/**
3993 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3994 * into the guest-state area in the VMCS.
3995 *
3996 * @returns VBox status code.
3997 * @param pVM Pointer to the VM.
3998 * @param pVCPU Pointer to the VMCPU.
3999 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4000 * out-of-sync. Make sure to update the required fields
4001 * before using them.
4002 *
4003 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4004 * @remarks No-long-jump zone!!!
4005 */
4006static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4007{
4008 int rc = VERR_INTERNAL_ERROR_5;
4009 PVM pVM = pVCpu->CTX_SUFF(pVM);
4010
4011 /*
4012 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4013 */
4014 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4015 {
4016 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4017 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4018 {
4019 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4020 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4021 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4022 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4023 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4024 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4025 }
4026
4027#ifdef VBOX_WITH_REM
4028 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4029 {
4030 Assert(pVM->hm.s.vmx.pRealModeTSS);
4031 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4032 if ( pVCpu->hm.s.vmx.fWasInRealMode
4033 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4034 {
4035 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4036 in real-mode (e.g. OpenBSD 4.0) */
4037 REMFlushTBs(pVM);
4038 Log4(("Load: Switch to protected mode detected!\n"));
4039 pVCpu->hm.s.vmx.fWasInRealMode = false;
4040 }
4041 }
4042#endif
4043 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4044 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
4045 AssertRCReturn(rc, rc);
4046 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4047 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
4048 AssertRCReturn(rc, rc);
4049 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4050 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
4051 AssertRCReturn(rc, rc);
4052 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4053 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
4054 AssertRCReturn(rc, rc);
4055 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4056 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
4057 AssertRCReturn(rc, rc);
4058 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4059 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
4060 AssertRCReturn(rc, rc);
4061
4062#ifdef VBOX_STRICT
4063 /* Validate. */
4064 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4065#endif
4066
4067 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4068 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4069 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4070 }
4071
4072 /*
4073 * Guest TR.
4074 */
4075 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4076 {
4077 /*
4078 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4079 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4080 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4081 */
4082 uint16_t u16Sel = 0;
4083 uint32_t u32Limit = 0;
4084 uint64_t u64Base = 0;
4085 uint32_t u32AccessRights = 0;
4086
4087 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4088 {
4089 u16Sel = pMixedCtx->tr.Sel;
4090 u32Limit = pMixedCtx->tr.u32Limit;
4091 u64Base = pMixedCtx->tr.u64Base;
4092 u32AccessRights = pMixedCtx->tr.Attr.u;
4093 }
4094 else
4095 {
4096 Assert(pVM->hm.s.vmx.pRealModeTSS);
4097 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4098
4099 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4100 RTGCPHYS GCPhys;
4101 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4102 AssertRCReturn(rc, rc);
4103
4104 X86DESCATTR DescAttr;
4105 DescAttr.u = 0;
4106 DescAttr.n.u1Present = 1;
4107 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4108
4109 u16Sel = 0;
4110 u32Limit = HM_VTX_TSS_SIZE;
4111 u64Base = GCPhys; /* in real-mode phys = virt. */
4112 u32AccessRights = DescAttr.u;
4113 }
4114
4115 /* Validate. */
4116 Assert(!(u16Sel & RT_BIT(2)));
4117 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4118 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4119 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4120 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4121 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4122 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4123 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4124 Assert( (u32Limit & 0xfff) == 0xfff
4125 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4126 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4127 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4128
4129 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4130 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4131 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4132 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4133
4134 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4135 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4136 }
4137
4138 /*
4139 * Guest GDTR.
4140 */
4141 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4142 {
4143 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4144 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4145
4146 /* Validate. */
4147 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4148
4149 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4150 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4151 }
4152
4153 /*
4154 * Guest LDTR.
4155 */
4156 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4157 {
4158 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4159 uint32_t u32Access = 0;
4160 if (!pMixedCtx->ldtr.Attr.u)
4161 u32Access = X86DESCATTR_UNUSABLE;
4162 else
4163 u32Access = pMixedCtx->ldtr.Attr.u;
4164
4165 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4166 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4167 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4168 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4169
4170 /* Validate. */
4171 if (!(u32Access & X86DESCATTR_UNUSABLE))
4172 {
4173 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4174 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4175 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4176 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4177 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4178 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4179 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4180 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4181 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4182 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4183 }
4184
4185 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4186 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4187 }
4188
4189 /*
4190 * Guest IDTR.
4191 */
4192 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4193 {
4194 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4195 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4196
4197 /* Validate. */
4198 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4199
4200 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4201 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4202 }
4203
4204 return VINF_SUCCESS;
4205}
4206
4207
4208/**
4209 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4210 * areas. These MSRs will automatically be loaded to the host CPU on every
4211 * successful VM entry and stored from the host CPU on every successful VM exit.
4212 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4213 *
4214 * @returns VBox status code.
4215 * @param pVCpu Pointer to the VMCPU.
4216 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4217 * out-of-sync. Make sure to update the required fields
4218 * before using them.
4219 *
4220 * @remarks No-long-jump zone!!!
4221 */
4222static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4223{
4224 AssertPtr(pVCpu);
4225 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4226
4227 /*
4228 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4229 */
4230 int rc = VINF_SUCCESS;
4231 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4232 {
4233#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4234 PVM pVM = pVCpu->CTX_SUFF(pVM);
4235 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4236 uint32_t cGuestMsrs = 0;
4237
4238 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4239 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4240 * when the guest really is in 64-bit mode. */
4241 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4242 if (fSupportsLongMode)
4243 {
4244 pGuestMsr->u32Msr = MSR_K8_LSTAR;
4245 pGuestMsr->u32Reserved = 0;
4246 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4247 pGuestMsr++; cGuestMsrs++;
4248 pGuestMsr->u32Msr = MSR_K6_STAR;
4249 pGuestMsr->u32Reserved = 0;
4250 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4251 pGuestMsr++; cGuestMsrs++;
4252 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4253 pGuestMsr->u32Reserved = 0;
4254 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4255 pGuestMsr++; cGuestMsrs++;
4256 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4257 pGuestMsr->u32Reserved = 0;
4258 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4259 pGuestMsr++; cGuestMsrs++;
4260 }
4261
4262 /*
4263 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4264 * load the guest's copy always (since the MSR bitmap allows passthru unconditionally).
4265 */
4266 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
4267 {
4268 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4269 pGuestMsr->u32Reserved = 0;
4270 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4271 AssertRCReturn(rc, rc);
4272 pGuestMsr++; cGuestMsrs++;
4273 }
4274
4275 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4276 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4277 {
4278 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4279 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4280 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4281 }
4282
4283 /* Update the VCPU's copy of the guest MSR count. */
4284 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4285 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4286 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4287#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4288
4289 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4290 }
4291
4292 /*
4293 * Guest Sysenter MSRs.
4294 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4295 * VM-exits on WRMSRs for these MSRs.
4296 */
4297 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4298 {
4299 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4300 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4301 }
4302
4303 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4304 {
4305 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4306 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4307 }
4308
4309 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4310 {
4311 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4312 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4313 }
4314
4315 return rc;
4316}
4317
4318
4319/**
4320 * Loads the guest activity state into the guest-state area in the VMCS.
4321 *
4322 * @returns VBox status code.
4323 * @param pVCpu Pointer to the VMCPU.
4324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4325 * out-of-sync. Make sure to update the required fields
4326 * before using them.
4327 *
4328 * @remarks No-long-jump zone!!!
4329 */
4330static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4331{
4332 /** @todo See if we can make use of other states, e.g.
4333 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4334 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4335 {
4336 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4337 AssertRCReturn(rc, rc);
4338
4339 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4340 }
4341 return VINF_SUCCESS;
4342}
4343
4344
4345/**
4346 * Sets up the appropriate function to run guest code.
4347 *
4348 * @returns VBox status code.
4349 * @param pVCpu Pointer to the VMCPU.
4350 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4351 * out-of-sync. Make sure to update the required fields
4352 * before using them.
4353 *
4354 * @remarks No-long-jump zone!!!
4355 */
4356static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4357{
4358 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4359 {
4360#ifndef VBOX_ENABLE_64_BITS_GUESTS
4361 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4362#endif
4363 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4364#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4365 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4366 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4367 {
4368 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4369 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4370 }
4371#else
4372 /* 64-bit host or hybrid host. */
4373 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4374#endif
4375 }
4376 else
4377 {
4378 /* Guest is not in long mode, use the 32-bit handler. */
4379#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4380 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4381 {
4382 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4383 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4384 }
4385#else
4386 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4387#endif
4388 }
4389 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4390 return VINF_SUCCESS;
4391}
4392
4393
4394/**
4395 * Wrapper for running the guest code in VT-x.
4396 *
4397 * @returns VBox strict status code.
4398 * @param pVM Pointer to the VM.
4399 * @param pVCpu Pointer to the VMCPU.
4400 * @param pCtx Pointer to the guest-CPU context.
4401 *
4402 * @remarks No-long-jump zone!!!
4403 */
4404DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4405{
4406 /*
4407 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4408 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4409 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4410 */
4411 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4412 /** @todo Add stats for resume vs launch. */
4413#ifdef VBOX_WITH_KERNEL_USING_XMM
4414 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4415#else
4416 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4417#endif
4418}
4419
4420
4421/**
4422 * Reports world-switch error and dumps some useful debug info.
4423 *
4424 * @param pVM Pointer to the VM.
4425 * @param pVCpu Pointer to the VMCPU.
4426 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4427 * @param pCtx Pointer to the guest-CPU context.
4428 * @param pVmxTransient Pointer to the VMX transient structure (only
4429 * exitReason updated).
4430 */
4431static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4432{
4433 Assert(pVM);
4434 Assert(pVCpu);
4435 Assert(pCtx);
4436 Assert(pVmxTransient);
4437 HMVMX_ASSERT_PREEMPT_SAFE();
4438
4439 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4440 switch (rcVMRun)
4441 {
4442 case VERR_VMX_INVALID_VMXON_PTR:
4443 AssertFailed();
4444 break;
4445 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4446 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4447 {
4448 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4449 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4450 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4451 AssertRC(rc);
4452
4453 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4454 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4455 Cannot do it here as we may have been long preempted. */
4456
4457#ifdef VBOX_STRICT
4458 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4459 pVmxTransient->uExitReason));
4460 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4461 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4462 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4463 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4464 else
4465 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4466 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4467 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4468
4469 /* VMX control bits. */
4470 uint32_t u32Val;
4471 uint64_t u64Val;
4472 HMVMXHCUINTREG uHCReg;
4473 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4474 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4475 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4476 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4477 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4478 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4479 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4480 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4481 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4482 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4483 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4484 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4485 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4486 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4487 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4488 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4489 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4490 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4491 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4492 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4493 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4494 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4495 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4496 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4497 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4498 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4499 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4500 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4501 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4502 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4503 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4504 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4505 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4506 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4507 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4508 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4509 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4510 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4511 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4512 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4513 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4514 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4515
4516 /* Guest bits. */
4517 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4518 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4519 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4520 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4521 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4522 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4523 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4524 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4525
4526 /* Host bits. */
4527 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4528 Log4(("Host CR0 %#RHr\n", uHCReg));
4529 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4530 Log4(("Host CR3 %#RHr\n", uHCReg));
4531 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4532 Log4(("Host CR4 %#RHr\n", uHCReg));
4533
4534 RTGDTR HostGdtr;
4535 PCX86DESCHC pDesc;
4536 ASMGetGDTR(&HostGdtr);
4537 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4538 Log4(("Host CS %#08x\n", u32Val));
4539 if (u32Val < HostGdtr.cbGdt)
4540 {
4541 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4542 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4543 }
4544
4545 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4546 Log4(("Host DS %#08x\n", u32Val));
4547 if (u32Val < HostGdtr.cbGdt)
4548 {
4549 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4550 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4551 }
4552
4553 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4554 Log4(("Host ES %#08x\n", u32Val));
4555 if (u32Val < HostGdtr.cbGdt)
4556 {
4557 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4558 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4559 }
4560
4561 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4562 Log4(("Host FS %#08x\n", u32Val));
4563 if (u32Val < HostGdtr.cbGdt)
4564 {
4565 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4566 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4567 }
4568
4569 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4570 Log4(("Host GS %#08x\n", u32Val));
4571 if (u32Val < HostGdtr.cbGdt)
4572 {
4573 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4574 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4575 }
4576
4577 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4578 Log4(("Host SS %#08x\n", u32Val));
4579 if (u32Val < HostGdtr.cbGdt)
4580 {
4581 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4582 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4583 }
4584
4585 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4586 Log4(("Host TR %#08x\n", u32Val));
4587 if (u32Val < HostGdtr.cbGdt)
4588 {
4589 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4590 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4591 }
4592
4593 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4594 Log4(("Host TR Base %#RHv\n", uHCReg));
4595 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4596 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4597 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4598 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4599 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4600 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4601 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4602 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4603 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4604 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4605 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4606 Log4(("Host RSP %#RHv\n", uHCReg));
4607 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4608 Log4(("Host RIP %#RHv\n", uHCReg));
4609# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4610 if (HMVMX_IS_64BIT_HOST_MODE())
4611 {
4612 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4613 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4614 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4615 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4616 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4617 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4618 }
4619# endif
4620#endif /* VBOX_STRICT */
4621 break;
4622 }
4623
4624 default:
4625 /* Impossible */
4626 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4627 break;
4628 }
4629 NOREF(pVM);
4630}
4631
4632
4633#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4634#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4635# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4636#endif
4637#ifdef VBOX_STRICT
4638static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4639{
4640 switch (idxField)
4641 {
4642 case VMX_VMCS_GUEST_RIP:
4643 case VMX_VMCS_GUEST_RSP:
4644 case VMX_VMCS_GUEST_SYSENTER_EIP:
4645 case VMX_VMCS_GUEST_SYSENTER_ESP:
4646 case VMX_VMCS_GUEST_GDTR_BASE:
4647 case VMX_VMCS_GUEST_IDTR_BASE:
4648 case VMX_VMCS_GUEST_CS_BASE:
4649 case VMX_VMCS_GUEST_DS_BASE:
4650 case VMX_VMCS_GUEST_ES_BASE:
4651 case VMX_VMCS_GUEST_FS_BASE:
4652 case VMX_VMCS_GUEST_GS_BASE:
4653 case VMX_VMCS_GUEST_SS_BASE:
4654 case VMX_VMCS_GUEST_LDTR_BASE:
4655 case VMX_VMCS_GUEST_TR_BASE:
4656 case VMX_VMCS_GUEST_CR3:
4657 return true;
4658 }
4659 return false;
4660}
4661
4662static bool hmR0VmxIsValidReadField(uint32_t idxField)
4663{
4664 switch (idxField)
4665 {
4666 /* Read-only fields. */
4667 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4668 return true;
4669 }
4670 /* Remaining readable fields should also be writable. */
4671 return hmR0VmxIsValidWriteField(idxField);
4672}
4673#endif /* VBOX_STRICT */
4674
4675
4676/**
4677 * Executes the specified handler in 64-bit mode.
4678 *
4679 * @returns VBox status code.
4680 * @param pVM Pointer to the VM.
4681 * @param pVCpu Pointer to the VMCPU.
4682 * @param pCtx Pointer to the guest CPU context.
4683 * @param enmOp The operation to perform.
4684 * @param cbParam Number of parameters.
4685 * @param paParam Array of 32-bit parameters.
4686 */
4687VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4688 uint32_t *paParam)
4689{
4690 int rc, rc2;
4691 PHMGLOBALCPUINFO pCpu;
4692 RTHCPHYS HCPhysCpuPage;
4693 RTCCUINTREG uOldEflags;
4694
4695 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4696 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4697 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4698 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4699
4700#ifdef VBOX_STRICT
4701 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4702 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4703
4704 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4705 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4706#endif
4707
4708 /* Disable interrupts. */
4709 uOldEflags = ASMIntDisableFlags();
4710
4711#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4712 RTCPUID idHostCpu = RTMpCpuId();
4713 CPUMR0SetLApic(pVCpu, idHostCpu);
4714#endif
4715
4716 pCpu = HMR0GetCurrentCpu();
4717 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4718
4719 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4720 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4721
4722 /* Leave VMX Root Mode. */
4723 VMXDisable();
4724
4725 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4726
4727 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4728 CPUMSetHyperEIP(pVCpu, enmOp);
4729 for (int i = (int)cbParam - 1; i >= 0; i--)
4730 CPUMPushHyper(pVCpu, paParam[i]);
4731
4732 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4733
4734 /* Call the switcher. */
4735 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4736 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4737
4738 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4739 /* Make sure the VMX instructions don't cause #UD faults. */
4740 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4741
4742 /* Re-enter VMX Root Mode */
4743 rc2 = VMXEnable(HCPhysCpuPage);
4744 if (RT_FAILURE(rc2))
4745 {
4746 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4747 ASMSetFlags(uOldEflags);
4748 return rc2;
4749 }
4750
4751 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4752 AssertRC(rc2);
4753 Assert(!(ASMGetFlags() & X86_EFL_IF));
4754 ASMSetFlags(uOldEflags);
4755 return rc;
4756}
4757
4758
4759/**
4760 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4761 * supporting 64-bit guests.
4762 *
4763 * @returns VBox status code.
4764 * @param fResume Whether to VMLAUNCH or VMRESUME.
4765 * @param pCtx Pointer to the guest-CPU context.
4766 * @param pCache Pointer to the VMCS cache.
4767 * @param pVM Pointer to the VM.
4768 * @param pVCpu Pointer to the VMCPU.
4769 */
4770DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4771{
4772 uint32_t aParam[6];
4773 PHMGLOBALCPUINFO pCpu = NULL;
4774 RTHCPHYS HCPhysCpuPage = 0;
4775 int rc = VERR_INTERNAL_ERROR_5;
4776
4777 pCpu = HMR0GetCurrentCpu();
4778 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4779
4780#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4781 pCache->uPos = 1;
4782 pCache->interPD = PGMGetInterPaeCR3(pVM);
4783 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4784#endif
4785
4786#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4787 pCache->TestIn.HCPhysCpuPage = 0;
4788 pCache->TestIn.HCPhysVmcs = 0;
4789 pCache->TestIn.pCache = 0;
4790 pCache->TestOut.HCPhysVmcs = 0;
4791 pCache->TestOut.pCache = 0;
4792 pCache->TestOut.pCtx = 0;
4793 pCache->TestOut.eflags = 0;
4794#endif
4795
4796 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4797 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4798 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4799 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4800 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4801 aParam[5] = 0;
4802
4803#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4804 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4805 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4806#endif
4807 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4808
4809#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4810 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4811 Assert(pCtx->dr[4] == 10);
4812 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4813#endif
4814
4815#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4816 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4817 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4818 pVCpu->hm.s.vmx.HCPhysVmcs));
4819 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4820 pCache->TestOut.HCPhysVmcs));
4821 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4822 pCache->TestOut.pCache));
4823 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4824 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4825 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4826 pCache->TestOut.pCtx));
4827 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4828#endif
4829 return rc;
4830}
4831
4832
4833/**
4834 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4835 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4836 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4837 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4838 *
4839 * @returns VBox status code.
4840 * @param pVM Pointer to the VM.
4841 * @param pVCpu Pointer to the VMCPU.
4842 */
4843static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4844{
4845#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4846{ \
4847 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4848 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4849 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4850 ++cReadFields; \
4851}
4852
4853 AssertPtr(pVM);
4854 AssertPtr(pVCpu);
4855 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4856 uint32_t cReadFields = 0;
4857
4858 /*
4859 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4860 * and serve to indicate exceptions to the rules.
4861 */
4862
4863 /* Guest-natural selector base fields. */
4864#if 0
4865 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4866 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4867 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4868#endif
4869 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4870 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4871 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4872 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4873 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4874 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4875 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4876 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4877 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4878 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4879 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4880 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4881#if 0
4882 /* Unused natural width guest-state fields. */
4883 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4884 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4885#endif
4886 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4887 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4888
4889 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4890#if 0
4891 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4892 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4893 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4894 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4895 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4896 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4897 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4898 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4899 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4900#endif
4901
4902 /* Natural width guest-state fields. */
4903 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4904#if 0
4905 /* Currently unused field. */
4906 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4907#endif
4908
4909 if (pVM->hm.s.fNestedPaging)
4910 {
4911 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4912 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4913 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4914 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4915 }
4916 else
4917 {
4918 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4919 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4920 }
4921
4922#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4923 return VINF_SUCCESS;
4924}
4925
4926
4927/**
4928 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4929 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4930 * darwin, running 64-bit guests).
4931 *
4932 * @returns VBox status code.
4933 * @param pVCpu Pointer to the VMCPU.
4934 * @param idxField The VMCS field encoding.
4935 * @param u64Val 16, 32 or 64 bits value.
4936 */
4937VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4938{
4939 int rc;
4940 switch (idxField)
4941 {
4942 /*
4943 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4944 */
4945 /* 64-bit Control fields. */
4946 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4947 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4948 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4949 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4950 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4951 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4952 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4953 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4954 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4955 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4956 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4957 case VMX_VMCS64_CTRL_EPTP_FULL:
4958 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4959 /* 64-bit Guest-state fields. */
4960 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4961 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4962 case VMX_VMCS64_GUEST_PAT_FULL:
4963 case VMX_VMCS64_GUEST_EFER_FULL:
4964 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4965 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4966 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4967 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4968 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4969 /* 64-bit Host-state fields. */
4970 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4971 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4972 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4973 {
4974 rc = VMXWriteVmcs32(idxField, u64Val);
4975 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4976 break;
4977 }
4978
4979 /*
4980 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4981 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4982 */
4983 /* Natural-width Guest-state fields. */
4984 case VMX_VMCS_GUEST_CR3:
4985 case VMX_VMCS_GUEST_ES_BASE:
4986 case VMX_VMCS_GUEST_CS_BASE:
4987 case VMX_VMCS_GUEST_SS_BASE:
4988 case VMX_VMCS_GUEST_DS_BASE:
4989 case VMX_VMCS_GUEST_FS_BASE:
4990 case VMX_VMCS_GUEST_GS_BASE:
4991 case VMX_VMCS_GUEST_LDTR_BASE:
4992 case VMX_VMCS_GUEST_TR_BASE:
4993 case VMX_VMCS_GUEST_GDTR_BASE:
4994 case VMX_VMCS_GUEST_IDTR_BASE:
4995 case VMX_VMCS_GUEST_RSP:
4996 case VMX_VMCS_GUEST_RIP:
4997 case VMX_VMCS_GUEST_SYSENTER_ESP:
4998 case VMX_VMCS_GUEST_SYSENTER_EIP:
4999 {
5000 if (!(u64Val >> 32))
5001 {
5002 /* If this field is 64-bit, VT-x will zero out the top bits. */
5003 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5004 }
5005 else
5006 {
5007 /* Assert that only the 32->64 switcher case should ever come here. */
5008 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5009 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5010 }
5011 break;
5012 }
5013
5014 default:
5015 {
5016 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5017 rc = VERR_INVALID_PARAMETER;
5018 break;
5019 }
5020 }
5021 AssertRCReturn(rc, rc);
5022 return rc;
5023}
5024
5025
5026/**
5027 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5028 * hosts (except darwin) for 64-bit guests.
5029 *
5030 * @param pVCpu Pointer to the VMCPU.
5031 * @param idxField The VMCS field encoding.
5032 * @param u64Val 16, 32 or 64 bits value.
5033 */
5034VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5035{
5036 AssertPtr(pVCpu);
5037 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5038
5039 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5040 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5041
5042 /* Make sure there are no duplicates. */
5043 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5044 {
5045 if (pCache->Write.aField[i] == idxField)
5046 {
5047 pCache->Write.aFieldVal[i] = u64Val;
5048 return VINF_SUCCESS;
5049 }
5050 }
5051
5052 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5053 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5054 pCache->Write.cValidEntries++;
5055 return VINF_SUCCESS;
5056}
5057
5058/* Enable later when the assembly code uses these as callbacks. */
5059#if 0
5060/*
5061 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5062 *
5063 * @param pVCpu Pointer to the VMCPU.
5064 * @param pCache Pointer to the VMCS cache.
5065 *
5066 * @remarks No-long-jump zone!!!
5067 */
5068VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5069{
5070 AssertPtr(pCache);
5071 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5072 {
5073 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5074 AssertRC(rc);
5075 }
5076 pCache->Write.cValidEntries = 0;
5077}
5078
5079
5080/**
5081 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5082 *
5083 * @param pVCpu Pointer to the VMCPU.
5084 * @param pCache Pointer to the VMCS cache.
5085 *
5086 * @remarks No-long-jump zone!!!
5087 */
5088VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5089{
5090 AssertPtr(pCache);
5091 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5092 {
5093 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5094 AssertRC(rc);
5095 }
5096}
5097#endif
5098#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5099
5100
5101/**
5102 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5103 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5104 * timer.
5105 *
5106 * @returns VBox status code.
5107 * @param pVCpu Pointer to the VMCPU.
5108 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5109 * out-of-sync. Make sure to update the required fields
5110 * before using them.
5111 * @remarks No-long-jump zone!!!
5112 */
5113static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5114{
5115 int rc = VERR_INTERNAL_ERROR_5;
5116 bool fOffsettedTsc = false;
5117 PVM pVM = pVCpu->CTX_SUFF(pVM);
5118 if (pVM->hm.s.vmx.fUsePreemptTimer)
5119 {
5120 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5121
5122 /* Make sure the returned values have sane upper and lower boundaries. */
5123 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5124 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5125 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5126 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5127
5128 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5129 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5130 }
5131 else
5132 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5133
5134 if (fOffsettedTsc)
5135 {
5136 uint64_t u64CurTSC = ASMReadTSC();
5137 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5138 {
5139 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5140 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5141
5142 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5143 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5144 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5145 }
5146 else
5147 {
5148 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5149 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5150 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5151 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5152 }
5153 }
5154 else
5155 {
5156 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5157 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5158 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5159 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5160 }
5161}
5162
5163
5164/**
5165 * Determines if an exception is a contributory exception. Contributory
5166 * exceptions are ones which can cause double-faults. Page-fault is
5167 * intentionally not included here as it's a conditional contributory exception.
5168 *
5169 * @returns true if the exception is contributory, false otherwise.
5170 * @param uVector The exception vector.
5171 */
5172DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5173{
5174 switch (uVector)
5175 {
5176 case X86_XCPT_GP:
5177 case X86_XCPT_SS:
5178 case X86_XCPT_NP:
5179 case X86_XCPT_TS:
5180 case X86_XCPT_DE:
5181 return true;
5182 default:
5183 break;
5184 }
5185 return false;
5186}
5187
5188
5189/**
5190 * Sets an event as a pending event to be injected into the guest.
5191 *
5192 * @param pVCpu Pointer to the VMCPU.
5193 * @param u32IntInfo The VM-entry interruption-information field.
5194 * @param cbInstr The VM-entry instruction length in bytes (for software
5195 * interrupts, exceptions and privileged software
5196 * exceptions).
5197 * @param u32ErrCode The VM-entry exception error code.
5198 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5199 * page-fault.
5200 *
5201 * @remarks Statistics counter assumes this is a guest event being injected or
5202 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5203 * always incremented.
5204 */
5205DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5206 RTGCUINTPTR GCPtrFaultAddress)
5207{
5208 Assert(!pVCpu->hm.s.Event.fPending);
5209 pVCpu->hm.s.Event.fPending = true;
5210 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5211 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5212 pVCpu->hm.s.Event.cbInstr = cbInstr;
5213 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5214
5215 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5216}
5217
5218
5219/**
5220 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5221 *
5222 * @param pVCpu Pointer to the VMCPU.
5223 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5224 * out-of-sync. Make sure to update the required fields
5225 * before using them.
5226 */
5227DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5228{
5229 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5230 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5231 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5232 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5233}
5234
5235
5236/**
5237 * Handle a condition that occurred while delivering an event through the guest
5238 * IDT.
5239 *
5240 * @returns VBox status code (informational error codes included).
5241 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5242 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5243 * continue execution of the guest which will delivery the #DF.
5244 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5245 *
5246 * @param pVCpu Pointer to the VMCPU.
5247 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5248 * out-of-sync. Make sure to update the required fields
5249 * before using them.
5250 * @param pVmxTransient Pointer to the VMX transient structure.
5251 *
5252 * @remarks No-long-jump zone!!!
5253 */
5254static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5255{
5256 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5257 AssertRCReturn(rc, rc);
5258 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5259 {
5260 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5261 AssertRCReturn(rc, rc);
5262
5263 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5264 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5265 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5266
5267 typedef enum
5268 {
5269 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5270 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5271 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5272 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5273 } VMXREFLECTXCPT;
5274
5275 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5276 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5277 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5278 {
5279 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5280 {
5281 enmReflect = VMXREFLECTXCPT_XCPT;
5282#ifdef VBOX_STRICT
5283 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5284 && uExitVector == X86_XCPT_PF)
5285 {
5286 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5287 }
5288#endif
5289 if ( uExitVector == X86_XCPT_PF
5290 && uIdtVector == X86_XCPT_PF)
5291 {
5292 pVmxTransient->fVectoringPF = true;
5293 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5294 }
5295 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5296 && hmR0VmxIsContributoryXcpt(uExitVector)
5297 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5298 || uIdtVector == X86_XCPT_PF))
5299 {
5300 enmReflect = VMXREFLECTXCPT_DF;
5301 }
5302 else if (uIdtVector == X86_XCPT_DF)
5303 enmReflect = VMXREFLECTXCPT_TF;
5304 }
5305 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5306 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5307 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5308 {
5309 /*
5310 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5311 * (whatever they are) as they reoccur when restarting the instruction.
5312 */
5313 enmReflect = VMXREFLECTXCPT_XCPT;
5314 }
5315 }
5316 else
5317 {
5318 /*
5319 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5320 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5321 * original exception to the guest after handling the VM-exit.
5322 */
5323 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5324 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5325 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5326 {
5327 enmReflect = VMXREFLECTXCPT_XCPT;
5328 }
5329 }
5330
5331 switch (enmReflect)
5332 {
5333 case VMXREFLECTXCPT_XCPT:
5334 {
5335 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5336 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5337 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5338
5339 uint32_t u32ErrCode = 0;
5340 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5341 {
5342 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5343 AssertRCReturn(rc, rc);
5344 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5345 }
5346
5347 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5348 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5349 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5350 rc = VINF_SUCCESS;
5351 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5352 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5353
5354 break;
5355 }
5356
5357 case VMXREFLECTXCPT_DF:
5358 {
5359 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5360 rc = VINF_HM_DOUBLE_FAULT;
5361 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5362 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5363
5364 break;
5365 }
5366
5367 case VMXREFLECTXCPT_TF:
5368 {
5369 rc = VINF_EM_RESET;
5370 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5371 uExitVector));
5372 break;
5373 }
5374
5375 default:
5376 Assert(rc == VINF_SUCCESS);
5377 break;
5378 }
5379 }
5380 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5381 return rc;
5382}
5383
5384
5385/**
5386 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5387 *
5388 * @returns VBox status code.
5389 * @param pVCpu Pointer to the VMCPU.
5390 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5391 * out-of-sync. Make sure to update the required fields
5392 * before using them.
5393 *
5394 * @remarks No-long-jump zone!!!
5395 */
5396static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5397{
5398 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5399 {
5400 uint32_t uVal = 0;
5401 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5402 AssertRCReturn(rc, rc);
5403
5404 uint32_t uShadow = 0;
5405 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5406 AssertRCReturn(rc, rc);
5407
5408 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5409 CPUMSetGuestCR0(pVCpu, uVal);
5410 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5411 }
5412 return VINF_SUCCESS;
5413}
5414
5415
5416/**
5417 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5418 *
5419 * @returns VBox status code.
5420 * @param pVCpu Pointer to the VMCPU.
5421 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5422 * out-of-sync. Make sure to update the required fields
5423 * before using them.
5424 *
5425 * @remarks No-long-jump zone!!!
5426 */
5427static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5428{
5429 int rc = VINF_SUCCESS;
5430 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5431 {
5432 uint32_t uVal = 0;
5433 uint32_t uShadow = 0;
5434 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5435 AssertRCReturn(rc, rc);
5436 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5437 AssertRCReturn(rc, rc);
5438
5439 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5440 CPUMSetGuestCR4(pVCpu, uVal);
5441 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5442 }
5443 return rc;
5444}
5445
5446
5447/**
5448 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5449 *
5450 * @returns VBox status code.
5451 * @param pVCpu Pointer to the VMCPU.
5452 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5453 * out-of-sync. Make sure to update the required fields
5454 * before using them.
5455 *
5456 * @remarks No-long-jump zone!!!
5457 */
5458static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5459{
5460 int rc = VINF_SUCCESS;
5461 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5462 {
5463 uint64_t u64Val = 0;
5464 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5465 AssertRCReturn(rc, rc);
5466
5467 pMixedCtx->rip = u64Val;
5468 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5469 }
5470 return rc;
5471}
5472
5473
5474/**
5475 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5476 *
5477 * @returns VBox status code.
5478 * @param pVCpu Pointer to the VMCPU.
5479 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5480 * out-of-sync. Make sure to update the required fields
5481 * before using them.
5482 *
5483 * @remarks No-long-jump zone!!!
5484 */
5485static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5486{
5487 int rc = VINF_SUCCESS;
5488 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5489 {
5490 uint64_t u64Val = 0;
5491 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5492 AssertRCReturn(rc, rc);
5493
5494 pMixedCtx->rsp = u64Val;
5495 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5496 }
5497 return rc;
5498}
5499
5500
5501/**
5502 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5503 *
5504 * @returns VBox status code.
5505 * @param pVCpu Pointer to the VMCPU.
5506 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5507 * out-of-sync. Make sure to update the required fields
5508 * before using them.
5509 *
5510 * @remarks No-long-jump zone!!!
5511 */
5512static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5513{
5514 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5515 {
5516 uint32_t uVal = 0;
5517 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5518 AssertRCReturn(rc, rc);
5519
5520 pMixedCtx->eflags.u32 = uVal;
5521 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5522 {
5523 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5524 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5525
5526 pMixedCtx->eflags.Bits.u1VM = 0;
5527 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5528 }
5529
5530 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5531 }
5532 return VINF_SUCCESS;
5533}
5534
5535
5536/**
5537 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5538 * guest-CPU context.
5539 */
5540DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5541{
5542 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5543 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5544 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5545 return rc;
5546}
5547
5548
5549/**
5550 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5551 * from the guest-state area in the VMCS.
5552 *
5553 * @param pVCpu Pointer to the VMCPU.
5554 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5555 * out-of-sync. Make sure to update the required fields
5556 * before using them.
5557 *
5558 * @remarks No-long-jump zone!!!
5559 */
5560static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5561{
5562 uint32_t uIntrState = 0;
5563 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5564 AssertRC(rc);
5565
5566 if (!uIntrState)
5567 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5568 else
5569 {
5570 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5571 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5572 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5573 AssertRC(rc);
5574 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5575 AssertRC(rc);
5576
5577 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5578 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5579 }
5580}
5581
5582
5583/**
5584 * Saves the guest's activity state.
5585 *
5586 * @returns VBox status code.
5587 * @param pVCpu Pointer to the VMCPU.
5588 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5589 * out-of-sync. Make sure to update the required fields
5590 * before using them.
5591 *
5592 * @remarks No-long-jump zone!!!
5593 */
5594static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5595{
5596 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5597 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5598 return VINF_SUCCESS;
5599}
5600
5601
5602/**
5603 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5604 * the current VMCS into the guest-CPU context.
5605 *
5606 * @returns VBox status code.
5607 * @param pVCpu Pointer to the VMCPU.
5608 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5609 * out-of-sync. Make sure to update the required fields
5610 * before using them.
5611 *
5612 * @remarks No-long-jump zone!!!
5613 */
5614static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5615{
5616 int rc = VINF_SUCCESS;
5617 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5618 {
5619 uint32_t u32Val = 0;
5620 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5621 pMixedCtx->SysEnter.cs = u32Val;
5622 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5623 }
5624
5625 uint64_t u64Val = 0;
5626 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5627 {
5628 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5629 pMixedCtx->SysEnter.eip = u64Val;
5630 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5631 }
5632 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5633 {
5634 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5635 pMixedCtx->SysEnter.esp = u64Val;
5636 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5637 }
5638 return rc;
5639}
5640
5641
5642/**
5643 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5644 * context.
5645 *
5646 * @returns VBox status code.
5647 * @param pVCpu Pointer to the VMCPU.
5648 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5649 * out-of-sync. Make sure to update the required fields
5650 * before using them.
5651 *
5652 * @remarks No-long-jump zone!!!
5653 */
5654static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5655{
5656 int rc = VINF_SUCCESS;
5657 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5658 {
5659 uint64_t u64Val = 0;
5660 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5661 pMixedCtx->fs.u64Base = u64Val;
5662 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5663 }
5664 return rc;
5665}
5666
5667
5668/**
5669 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5670 * context.
5671 *
5672 * @returns VBox status code.
5673 * @param pVCpu Pointer to the VMCPU.
5674 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5675 * out-of-sync. Make sure to update the required fields
5676 * before using them.
5677 *
5678 * @remarks No-long-jump zone!!!
5679 */
5680static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5681{
5682 int rc = VINF_SUCCESS;
5683 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5684 {
5685 uint64_t u64Val = 0;
5686 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5687 pMixedCtx->gs.u64Base = u64Val;
5688 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5689 }
5690 return rc;
5691}
5692
5693
5694/**
5695 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5696 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5697 * and TSC_AUX.
5698 *
5699 * @returns VBox status code.
5700 * @param pVCpu Pointer to the VMCPU.
5701 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5702 * out-of-sync. Make sure to update the required fields
5703 * before using them.
5704 *
5705 * @remarks No-long-jump zone!!!
5706 */
5707static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5708{
5709 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5710 return VINF_SUCCESS;
5711
5712#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5713 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5714 {
5715 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5716 pMsr += i;
5717 switch (pMsr->u32Msr)
5718 {
5719 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5720 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5721 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5722 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5723 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5724 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5725 default:
5726 {
5727 AssertFailed();
5728 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5729 }
5730 }
5731 }
5732#endif
5733
5734 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5735 return VINF_SUCCESS;
5736}
5737
5738
5739/**
5740 * Saves the guest control registers from the current VMCS into the guest-CPU
5741 * context.
5742 *
5743 * @returns VBox status code.
5744 * @param pVCpu Pointer to the VMCPU.
5745 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5746 * out-of-sync. Make sure to update the required fields
5747 * before using them.
5748 *
5749 * @remarks No-long-jump zone!!!
5750 */
5751static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5752{
5753 /* Guest CR0. Guest FPU. */
5754 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5755 AssertRCReturn(rc, rc);
5756
5757 /* Guest CR4. */
5758 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5759 AssertRCReturn(rc, rc);
5760
5761 /* Guest CR2 - updated always during the world-switch or in #PF. */
5762 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5763 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5764 {
5765 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5766 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5767
5768 PVM pVM = pVCpu->CTX_SUFF(pVM);
5769 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5770 || ( pVM->hm.s.fNestedPaging
5771 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5772 {
5773 uint64_t u64Val = 0;
5774 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5775 if (pMixedCtx->cr3 != u64Val)
5776 {
5777 CPUMSetGuestCR3(pVCpu, u64Val);
5778 if (VMMRZCallRing3IsEnabled(pVCpu))
5779 {
5780 PGMUpdateCR3(pVCpu, u64Val);
5781 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5782 }
5783 else
5784 {
5785 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5786 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5787 }
5788 }
5789
5790 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5791 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5792 {
5793 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5794 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5795 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5796 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5797
5798 if (VMMRZCallRing3IsEnabled(pVCpu))
5799 {
5800 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5801 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5802 }
5803 else
5804 {
5805 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5806 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5807 }
5808 }
5809 }
5810
5811 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5812 }
5813
5814 /*
5815 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5816 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5817 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5818 *
5819 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5820 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5821 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5822 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5823 *
5824 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5825 */
5826 if (VMMRZCallRing3IsEnabled(pVCpu))
5827 {
5828 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5829 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5830
5831 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5832 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5833
5834 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5835 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5836 }
5837
5838 return rc;
5839}
5840
5841
5842/**
5843 * Reads a guest segment register from the current VMCS into the guest-CPU
5844 * context.
5845 *
5846 * @returns VBox status code.
5847 * @param pVCpu Pointer to the VMCPU.
5848 * @param idxSel Index of the selector in the VMCS.
5849 * @param idxLimit Index of the segment limit in the VMCS.
5850 * @param idxBase Index of the segment base in the VMCS.
5851 * @param idxAccess Index of the access rights of the segment in the VMCS.
5852 * @param pSelReg Pointer to the segment selector.
5853 *
5854 * @remarks No-long-jump zone!!!
5855 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5856 * macro as that takes care of whether to read from the VMCS cache or
5857 * not.
5858 */
5859DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5860 PCPUMSELREG pSelReg)
5861{
5862 uint32_t u32Val = 0;
5863 int rc = VMXReadVmcs32(idxSel, &u32Val);
5864 AssertRCReturn(rc, rc);
5865 pSelReg->Sel = (uint16_t)u32Val;
5866 pSelReg->ValidSel = (uint16_t)u32Val;
5867 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5868
5869 rc = VMXReadVmcs32(idxLimit, &u32Val);
5870 AssertRCReturn(rc, rc);
5871 pSelReg->u32Limit = u32Val;
5872
5873 uint64_t u64Val = 0;
5874 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5875 AssertRCReturn(rc, rc);
5876 pSelReg->u64Base = u64Val;
5877
5878 rc = VMXReadVmcs32(idxAccess, &u32Val);
5879 AssertRCReturn(rc, rc);
5880 pSelReg->Attr.u = u32Val;
5881
5882 /*
5883 * If VT-x marks the segment as unusable, most other bits remain undefined:
5884 * - For CS the L, D and G bits have meaning.
5885 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5886 * - For the remaining data segments no bits are defined.
5887 *
5888 * The present bit and the unusable bit has been observed to be set at the
5889 * same time (the selector was supposed to invalid as we started executing
5890 * a V8086 interrupt in ring-0).
5891 *
5892 * What should be important for the rest of the VBox code that the P bit is
5893 * cleared. Some of the other VBox code recognizes the unusable bit, but
5894 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5895 * safe side here, we'll strip off P and other bits we don't care about. If
5896 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5897 *
5898 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5899 */
5900 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5901 {
5902 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5903
5904 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5905 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5906 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5907
5908 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5909#ifdef DEBUG_bird
5910 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5911 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5912 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5913#endif
5914 }
5915 return VINF_SUCCESS;
5916}
5917
5918
5919#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5920# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5921 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5922 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5923#else
5924# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5925 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5926 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5927#endif
5928
5929
5930/**
5931 * Saves the guest segment registers from the current VMCS into the guest-CPU
5932 * context.
5933 *
5934 * @returns VBox status code.
5935 * @param pVCpu Pointer to the VMCPU.
5936 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5937 * out-of-sync. Make sure to update the required fields
5938 * before using them.
5939 *
5940 * @remarks No-long-jump zone!!!
5941 */
5942static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5943{
5944 /* Guest segment registers. */
5945 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5946 {
5947 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5948 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5949 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5950 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5951 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5952 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5953 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5954
5955 /* Restore segment attributes for real-on-v86 mode hack. */
5956 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5957 {
5958 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5959 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5960 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5961 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5962 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5963 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5964 }
5965 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5966 }
5967
5968 return VINF_SUCCESS;
5969}
5970
5971
5972/**
5973 * Saves the guest descriptor table registers and task register from the current
5974 * VMCS into the guest-CPU context.
5975 *
5976 * @returns VBox status code.
5977 * @param pVCpu Pointer to the VMCPU.
5978 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5979 * out-of-sync. Make sure to update the required fields
5980 * before using them.
5981 *
5982 * @remarks No-long-jump zone!!!
5983 */
5984static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5985{
5986 int rc = VINF_SUCCESS;
5987
5988 /* Guest LDTR. */
5989 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5990 {
5991 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5992 AssertRCReturn(rc, rc);
5993 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5994 }
5995
5996 /* Guest GDTR. */
5997 uint64_t u64Val = 0;
5998 uint32_t u32Val = 0;
5999 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6000 {
6001 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6002 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6003 pMixedCtx->gdtr.pGdt = u64Val;
6004 pMixedCtx->gdtr.cbGdt = u32Val;
6005 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
6006 }
6007
6008 /* Guest IDTR. */
6009 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6010 {
6011 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6012 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6013 pMixedCtx->idtr.pIdt = u64Val;
6014 pMixedCtx->idtr.cbIdt = u32Val;
6015 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
6016 }
6017
6018 /* Guest TR. */
6019 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6020 {
6021 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6022 AssertRCReturn(rc, rc);
6023
6024 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6025 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6026 {
6027 rc = VMXLOCAL_READ_SEG(TR, tr);
6028 AssertRCReturn(rc, rc);
6029 }
6030 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
6031 }
6032 return rc;
6033}
6034
6035#undef VMXLOCAL_READ_SEG
6036
6037
6038/**
6039 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6040 * context.
6041 *
6042 * @returns VBox status code.
6043 * @param pVCpu Pointer to the VMCPU.
6044 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6045 * out-of-sync. Make sure to update the required fields
6046 * before using them.
6047 *
6048 * @remarks No-long-jump zone!!!
6049 */
6050static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6051{
6052 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6053 {
6054 if (!pVCpu->hm.s.fUsingHyperDR7)
6055 {
6056 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6057 uint32_t u32Val;
6058 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6059 pMixedCtx->dr[7] = u32Val;
6060 }
6061
6062 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
6063 }
6064 return VINF_SUCCESS;
6065}
6066
6067
6068/**
6069 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6070 *
6071 * @returns VBox status code.
6072 * @param pVCpu Pointer to the VMCPU.
6073 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6074 * out-of-sync. Make sure to update the required fields
6075 * before using them.
6076 *
6077 * @remarks No-long-jump zone!!!
6078 */
6079static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6080{
6081 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6082 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
6083 return VINF_SUCCESS;
6084}
6085
6086
6087/**
6088 * Saves the entire guest state from the currently active VMCS into the
6089 * guest-CPU context. This essentially VMREADs all guest-data.
6090 *
6091 * @returns VBox status code.
6092 * @param pVCpu Pointer to the VMCPU.
6093 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6094 * out-of-sync. Make sure to update the required fields
6095 * before using them.
6096 */
6097static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6098{
6099 Assert(pVCpu);
6100 Assert(pMixedCtx);
6101
6102 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6103 return VINF_SUCCESS;
6104
6105 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6106 again on the ring-3 callback path, there is no real need to. */
6107 if (VMMRZCallRing3IsEnabled(pVCpu))
6108 VMMR0LogFlushDisable(pVCpu);
6109 else
6110 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6111 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6112
6113 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6114 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6115
6116 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6117 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6118
6119 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6120 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6121
6122 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6123 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6124
6125 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6126 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6127
6128 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6129 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6130
6131 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
6132 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6133
6134 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
6135 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6136
6137 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6138 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6139
6140 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6141 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6142
6143 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6144 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6145
6146 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6147 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6148
6149 if (VMMRZCallRing3IsEnabled(pVCpu))
6150 VMMR0LogFlushEnable(pVCpu);
6151
6152 return rc;
6153}
6154
6155
6156/**
6157 * Check per-VM and per-VCPU force flag actions that require us to go back to
6158 * ring-3 for one reason or another.
6159 *
6160 * @returns VBox status code (information status code included).
6161 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6162 * ring-3.
6163 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6164 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6165 * interrupts)
6166 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6167 * all EMTs to be in ring-3.
6168 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6169 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6170 * to the EM loop.
6171 *
6172 * @param pVM Pointer to the VM.
6173 * @param pVCpu Pointer to the VMCPU.
6174 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6175 * out-of-sync. Make sure to update the required fields
6176 * before using them.
6177 */
6178static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6179{
6180 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6181
6182 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6183 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6184 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6185 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6186 {
6187 /* We need the control registers now, make sure the guest-CPU context is updated. */
6188 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6189 AssertRCReturn(rc3, rc3);
6190
6191 /* Pending HM CR3 sync. */
6192 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6193 {
6194 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6195 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6196 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6197 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6198 }
6199
6200 /* Pending HM PAE PDPEs. */
6201 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6202 {
6203 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6204 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6205 }
6206
6207 /* Pending PGM C3 sync. */
6208 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6209 {
6210 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6211 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6212 if (rc2 != VINF_SUCCESS)
6213 {
6214 AssertRC(rc2);
6215 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6216 return rc2;
6217 }
6218 }
6219
6220 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6221 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6222 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6223 {
6224 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6225 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6226 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6227 return rc2;
6228 }
6229
6230 /* Pending VM request packets, such as hardware interrupts. */
6231 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6232 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6233 {
6234 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6235 return VINF_EM_PENDING_REQUEST;
6236 }
6237
6238 /* Pending PGM pool flushes. */
6239 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6240 {
6241 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6242 return VINF_PGM_POOL_FLUSH_PENDING;
6243 }
6244
6245 /* Pending DMA requests. */
6246 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6247 {
6248 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6249 return VINF_EM_RAW_TO_R3;
6250 }
6251 }
6252
6253 return VINF_SUCCESS;
6254}
6255
6256
6257/**
6258 * Converts any TRPM trap into a pending HM event. This is typically used when
6259 * entering from ring-3 (not longjmp returns).
6260 *
6261 * @param pVCpu Pointer to the VMCPU.
6262 */
6263static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6264{
6265 Assert(TRPMHasTrap(pVCpu));
6266 Assert(!pVCpu->hm.s.Event.fPending);
6267
6268 uint8_t uVector;
6269 TRPMEVENT enmTrpmEvent;
6270 RTGCUINT uErrCode;
6271 RTGCUINTPTR GCPtrFaultAddress;
6272 uint8_t cbInstr;
6273
6274 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6275 AssertRC(rc);
6276
6277 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6278 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6279 if (enmTrpmEvent == TRPM_TRAP)
6280 {
6281 switch (uVector)
6282 {
6283 case X86_XCPT_BP:
6284 case X86_XCPT_OF:
6285 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6286 break;
6287
6288 case X86_XCPT_PF:
6289 case X86_XCPT_DF:
6290 case X86_XCPT_TS:
6291 case X86_XCPT_NP:
6292 case X86_XCPT_SS:
6293 case X86_XCPT_GP:
6294 case X86_XCPT_AC:
6295 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6296 /* no break! */
6297 default:
6298 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6299 break;
6300 }
6301 }
6302 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6303 {
6304 if (uVector == X86_XCPT_NMI)
6305 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6306 else
6307 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6308 }
6309 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6310 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6311 else
6312 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6313
6314 rc = TRPMResetTrap(pVCpu);
6315 AssertRC(rc);
6316 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6317 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6318
6319 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6320 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6321}
6322
6323
6324/**
6325 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6326 * VT-x to execute any instruction.
6327 *
6328 * @param pvCpu Pointer to the VMCPU.
6329 */
6330static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6331{
6332 Assert(pVCpu->hm.s.Event.fPending);
6333
6334 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6335 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6336 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6337 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6338
6339 /* If a trap was already pending, we did something wrong! */
6340 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6341
6342 TRPMEVENT enmTrapType;
6343 switch (uVectorType)
6344 {
6345 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6346 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6347 enmTrapType = TRPM_HARDWARE_INT;
6348 break;
6349
6350 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6351 enmTrapType = TRPM_SOFTWARE_INT;
6352 break;
6353
6354 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6355 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6356 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6357 enmTrapType = TRPM_TRAP;
6358 break;
6359
6360 default:
6361 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6362 enmTrapType = TRPM_32BIT_HACK;
6363 break;
6364 }
6365
6366 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6367
6368 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6369 AssertRC(rc);
6370
6371 if (fErrorCodeValid)
6372 TRPMSetErrorCode(pVCpu, uErrorCode);
6373
6374 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6375 && uVector == X86_XCPT_PF)
6376 {
6377 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6378 }
6379 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6380 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6381 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6382 {
6383 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6384 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6385 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6386 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6387 }
6388 pVCpu->hm.s.Event.fPending = false;
6389}
6390
6391
6392/**
6393 * Does the necessary state syncing before returning to ring-3 for any reason
6394 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6395 *
6396 * @returns VBox status code.
6397 * @param pVM Pointer to the VM.
6398 * @param pVCpu Pointer to the VMCPU.
6399 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6400 * be out-of-sync. Make sure to update the required
6401 * fields before using them.
6402 * @param fSaveGuestState Whether to save the guest state or not.
6403 *
6404 * @remarks If you modify code here, make sure to check whether
6405 * hmR0VmxCallRing3Callback() needs to be updated too.
6406 * @remarks No-long-jmp zone!!!
6407 */
6408static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6409{
6410 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6411 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6412
6413 RTCPUID idCpu = RTMpCpuId();
6414 Log4Func(("HostCpuId=%u\n", idCpu));
6415
6416 /* Save the guest state if necessary. */
6417 if ( fSaveGuestState
6418 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6419 {
6420 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6421 AssertRCReturn(rc, rc);
6422 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6423 }
6424
6425 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6426 if (CPUMIsGuestFPUStateActive(pVCpu))
6427 {
6428 /* We shouldn't reload CR0 without saving it first. */
6429 if (!fSaveGuestState)
6430 {
6431 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6432 AssertRCReturn(rc, rc);
6433 }
6434 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6435 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6436 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6437 }
6438
6439 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6440#ifdef VBOX_STRICT
6441 if (CPUMIsHyperDebugStateActive(pVCpu))
6442 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6443#endif
6444 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6445 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6446 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6447 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6448
6449#if HC_ARCH_BITS == 64
6450 /* Restore host-state bits that VT-x only restores partially. */
6451 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6452 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6453 {
6454 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6455 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6456 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6457 }
6458#endif
6459
6460 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6461 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6462 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6463 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6464 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6465 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6466 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6467 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6468
6469 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6470
6471 /** @todo This kinda defeats the purpose of having preemption hooks.
6472 * The problem is, deregistering the hooks should be moved to a place that
6473 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6474 * context.
6475 */
6476 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6477 {
6478 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6479 AssertRCReturn(rc, rc);
6480
6481 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6482 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6483 }
6484 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6485 NOREF(idCpu);
6486
6487 return VINF_SUCCESS;
6488}
6489
6490
6491/**
6492 * Leaves the VT-x session.
6493 *
6494 * @returns VBox status code.
6495 * @param pVM Pointer to the VM.
6496 * @param pVCpu Pointer to the VMCPU.
6497 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6498 * out-of-sync. Make sure to update the required fields
6499 * before using them.
6500 *
6501 * @remarks No-long-jmp zone!!!
6502 */
6503DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6504{
6505 HM_DISABLE_PREEMPT_IF_NEEDED();
6506 HMVMX_ASSERT_CPU_SAFE();
6507 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6508 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6509
6510 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6511 and done this from the VMXR0ThreadCtxCallback(). */
6512 if (!pVCpu->hm.s.fLeaveDone)
6513 {
6514 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6515 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6516 pVCpu->hm.s.fLeaveDone = true;
6517 }
6518
6519 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6520 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6521 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6522 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6523 VMMR0ThreadCtxHooksDeregister(pVCpu);
6524
6525 /* Leave HM context. This takes care of local init (term). */
6526 int rc = HMR0LeaveCpu(pVCpu);
6527
6528 HM_RESTORE_PREEMPT_IF_NEEDED();
6529
6530 return rc;
6531}
6532
6533
6534/**
6535 * Does the necessary state syncing before doing a longjmp to ring-3.
6536 *
6537 * @returns VBox status code.
6538 * @param pVM Pointer to the VM.
6539 * @param pVCpu Pointer to the VMCPU.
6540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6541 * out-of-sync. Make sure to update the required fields
6542 * before using them.
6543 *
6544 * @remarks No-long-jmp zone!!!
6545 */
6546DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6547{
6548 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6549}
6550
6551
6552/**
6553 * Take necessary actions before going back to ring-3.
6554 *
6555 * An action requires us to go back to ring-3. This function does the necessary
6556 * steps before we can safely return to ring-3. This is not the same as longjmps
6557 * to ring-3, this is voluntary and prepares the guest so it may continue
6558 * executing outside HM (recompiler/IEM).
6559 *
6560 * @returns VBox status code.
6561 * @param pVM Pointer to the VM.
6562 * @param pVCpu Pointer to the VMCPU.
6563 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6564 * out-of-sync. Make sure to update the required fields
6565 * before using them.
6566 * @param rcExit The reason for exiting to ring-3. Can be
6567 * VINF_VMM_UNKNOWN_RING3_CALL.
6568 */
6569static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6570{
6571 Assert(pVM);
6572 Assert(pVCpu);
6573 Assert(pMixedCtx);
6574 HMVMX_ASSERT_PREEMPT_SAFE();
6575
6576 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6577 {
6578 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6579 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6580 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6581 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6582 }
6583
6584 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6585 VMMRZCallRing3Disable(pVCpu);
6586 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6587
6588 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6589 if (pVCpu->hm.s.Event.fPending)
6590 {
6591 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6592 Assert(!pVCpu->hm.s.Event.fPending);
6593 }
6594
6595 /* Save guest state and restore host state bits. */
6596 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6597 AssertRCReturn(rc, rc);
6598 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6599
6600 /* Sync recompiler state. */
6601 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6602 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6603 | CPUM_CHANGED_LDTR
6604 | CPUM_CHANGED_GDTR
6605 | CPUM_CHANGED_IDTR
6606 | CPUM_CHANGED_TR
6607 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6608 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6609 if ( pVM->hm.s.fNestedPaging
6610 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6611 {
6612 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6613 }
6614
6615 Assert(!pVCpu->hm.s.fClearTrapFlag);
6616
6617 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6618 if (rcExit != VINF_EM_RAW_INTERRUPT)
6619 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6620
6621 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6622
6623 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6624 VMMRZCallRing3RemoveNotification(pVCpu);
6625 VMMRZCallRing3Enable(pVCpu);
6626
6627 return rc;
6628}
6629
6630
6631/**
6632 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6633 * longjump to ring-3 and possibly get preempted.
6634 *
6635 * @returns VBox status code.
6636 * @param pVCpu Pointer to the VMCPU.
6637 * @param enmOperation The operation causing the ring-3 longjump.
6638 * @param pvUser Opaque pointer to the guest-CPU context. The data
6639 * may be out-of-sync. Make sure to update the required
6640 * fields before using them.
6641 * @remarks If you modify code here, make sure to check whether
6642 * hmR0VmxLeave() needs to be updated too.
6643 */
6644DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6645{
6646 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6647 {
6648 VMMRZCallRing3RemoveNotification(pVCpu);
6649 HM_DISABLE_PREEMPT_IF_NEEDED();
6650
6651 /* If anything here asserts or fails, good luck. */
6652 if (CPUMIsGuestFPUStateActive(pVCpu))
6653 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6654
6655 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6656
6657#if HC_ARCH_BITS == 64
6658 /* Restore host-state bits that VT-x only restores partially. */
6659 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6660 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6661 {
6662 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6663 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6664 }
6665#endif
6666 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6667 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6668 {
6669 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6670 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6671 }
6672
6673 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6674 VMMR0ThreadCtxHooksDeregister(pVCpu);
6675
6676 HMR0LeaveCpu(pVCpu);
6677 HM_RESTORE_PREEMPT_IF_NEEDED();
6678 return VINF_SUCCESS;
6679 }
6680
6681 Assert(pVCpu);
6682 Assert(pvUser);
6683 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6684 HMVMX_ASSERT_PREEMPT_SAFE();
6685
6686 VMMRZCallRing3Disable(pVCpu);
6687 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6688
6689 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6690 enmOperation));
6691
6692 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6693 AssertRCReturn(rc, rc);
6694
6695 VMMRZCallRing3Enable(pVCpu);
6696 return VINF_SUCCESS;
6697}
6698
6699
6700/**
6701 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6702 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6703 *
6704 * @param pVCpu Pointer to the VMCPU.
6705 */
6706DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6707{
6708 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6709 {
6710 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6711 {
6712 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6713 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6714 AssertRC(rc);
6715 }
6716 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6717}
6718
6719
6720/**
6721 * Evaluates the event to be delivered to the guest and sets it as the pending
6722 * event.
6723 *
6724 * @param pVCpu Pointer to the VMCPU.
6725 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6726 * out-of-sync. Make sure to update the required fields
6727 * before using them.
6728 */
6729static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6730{
6731 Assert(!pVCpu->hm.s.Event.fPending);
6732
6733 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6734 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6735 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6736 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6737
6738 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6739 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6740 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6741 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6742 Assert(!TRPMHasTrap(pVCpu));
6743
6744 /** @todo SMI. SMIs take priority over NMIs. */
6745 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6746 {
6747 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6748 if ( !fBlockMovSS
6749 && !fBlockSti)
6750 {
6751 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6752 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6753 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6754 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6755
6756 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6757 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6758 }
6759 else
6760 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6761 }
6762 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6763 && !pVCpu->hm.s.fSingleInstruction)
6764 {
6765 /*
6766 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6767 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6768 * evaluated here and not set as pending, solely based on the force-flags.
6769 */
6770 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6771 AssertRC(rc);
6772 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6773 if ( !fBlockInt
6774 && !fBlockSti
6775 && !fBlockMovSS)
6776 {
6777 uint8_t u8Interrupt;
6778 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6779 if (RT_SUCCESS(rc))
6780 {
6781 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6782 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6783 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6784
6785 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6786 }
6787 else
6788 {
6789 /** @todo Does this actually happen? If not turn it into an assertion. */
6790 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6791 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6792 }
6793 }
6794 else
6795 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6796 }
6797}
6798
6799
6800/**
6801 * Injects any pending events into the guest if the guest is in a state to
6802 * receive them.
6803 *
6804 * @returns VBox status code (informational status codes included).
6805 * @param pVCpu Pointer to the VMCPU.
6806 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6807 * out-of-sync. Make sure to update the required fields
6808 * before using them.
6809 */
6810static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6811{
6812 HMVMX_ASSERT_PREEMPT_SAFE();
6813 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6814
6815 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6816 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6817 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6818 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6819
6820 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6821 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6822 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6823 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6824 Assert(!TRPMHasTrap(pVCpu));
6825
6826 int rc = VINF_SUCCESS;
6827 if (pVCpu->hm.s.Event.fPending)
6828 {
6829#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6830 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6831 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6832 {
6833 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6834 AssertRCReturn(rc, rc);
6835 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6836 Assert(!fBlockInt);
6837 Assert(!fBlockSti);
6838 Assert(!fBlockMovSS);
6839 }
6840 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6841 {
6842 Assert(!fBlockSti);
6843 Assert(!fBlockMovSS);
6844 }
6845#endif
6846 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6847 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6848 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6849 AssertRCReturn(rc, rc);
6850
6851 /* Update the interruptibility-state as it could have been changed by
6852 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6853 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6854 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6855
6856#ifdef VBOX_WITH_STATISTICS
6857 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6858 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6859 else
6860 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6861#endif
6862 }
6863
6864 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6865 int rc2 = VINF_SUCCESS;
6866 if ( fBlockSti
6867 || fBlockMovSS)
6868 {
6869 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6870 {
6871 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6872 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6873 {
6874 /*
6875 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6876 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6877 * See Intel spec. 27.3.4 "Saving Non-Register State".
6878 */
6879 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6880 AssertRCReturn(rc, rc);
6881 }
6882 }
6883 else
6884 {
6885 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6886 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6887 uIntrState = 0;
6888 }
6889 }
6890
6891 /*
6892 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6893 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6894 */
6895 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6896 AssertRC(rc2);
6897
6898 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6899 return rc;
6900}
6901
6902
6903/**
6904 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6905 *
6906 * @param pVCpu Pointer to the VMCPU.
6907 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6908 * out-of-sync. Make sure to update the required fields
6909 * before using them.
6910 */
6911DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6912{
6913 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6914 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6915}
6916
6917
6918/**
6919 * Injects a double-fault (#DF) exception into the VM.
6920 *
6921 * @returns VBox status code (informational status code included).
6922 * @param pVCpu Pointer to the VMCPU.
6923 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6924 * out-of-sync. Make sure to update the required fields
6925 * before using them.
6926 */
6927DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6928{
6929 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6930 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6931 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6932 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6933 puIntrState);
6934}
6935
6936
6937/**
6938 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6939 *
6940 * @param pVCpu Pointer to the VMCPU.
6941 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6942 * out-of-sync. Make sure to update the required fields
6943 * before using them.
6944 */
6945DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6946{
6947 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6948 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6949 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6950}
6951
6952
6953/**
6954 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6955 *
6956 * @param pVCpu Pointer to the VMCPU.
6957 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6958 * out-of-sync. Make sure to update the required fields
6959 * before using them.
6960 * @param cbInstr The value of RIP that is to be pushed on the guest
6961 * stack.
6962 */
6963DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6964{
6965 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6966 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6967 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6968}
6969
6970
6971/**
6972 * Injects a general-protection (#GP) fault into the VM.
6973 *
6974 * @returns VBox status code (informational status code included).
6975 * @param pVCpu Pointer to the VMCPU.
6976 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6977 * out-of-sync. Make sure to update the required fields
6978 * before using them.
6979 * @param u32ErrorCode The error code associated with the #GP.
6980 */
6981DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6982 uint32_t *puIntrState)
6983{
6984 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6985 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6986 if (fErrorCodeValid)
6987 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6988 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6989 puIntrState);
6990}
6991
6992
6993/**
6994 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6995 *
6996 * @param pVCpu Pointer to the VMCPU.
6997 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6998 * out-of-sync. Make sure to update the required fields
6999 * before using them.
7000 * @param uVector The software interrupt vector number.
7001 * @param cbInstr The value of RIP that is to be pushed on the guest
7002 * stack.
7003 */
7004DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7005{
7006 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7007 if ( uVector == X86_XCPT_BP
7008 || uVector == X86_XCPT_OF)
7009 {
7010 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7011 }
7012 else
7013 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7014 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7015}
7016
7017
7018/**
7019 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7020 * stack.
7021 *
7022 * @returns VBox status code (information status code included).
7023 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7024 * @param pVM Pointer to the VM.
7025 * @param pMixedCtx Pointer to the guest-CPU context.
7026 * @param uValue The value to push to the guest stack.
7027 */
7028DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7029{
7030 /*
7031 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7032 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7033 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7034 */
7035 if (pMixedCtx->sp == 1)
7036 return VINF_EM_RESET;
7037 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7038 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7039 AssertRCReturn(rc, rc);
7040 return rc;
7041}
7042
7043
7044/**
7045 * Injects an event into the guest upon VM-entry by updating the relevant fields
7046 * in the VM-entry area in the VMCS.
7047 *
7048 * @returns VBox status code (informational error codes included).
7049 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7050 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7051 *
7052 * @param pVCpu Pointer to the VMCPU.
7053 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7054 * be out-of-sync. Make sure to update the required
7055 * fields before using them.
7056 * @param u64IntInfo The VM-entry interruption-information field.
7057 * @param cbInstr The VM-entry instruction length in bytes (for
7058 * software interrupts, exceptions and privileged
7059 * software exceptions).
7060 * @param u32ErrCode The VM-entry exception error code.
7061 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7062 * @param puIntrState Pointer to the current guest interruptibility-state.
7063 * This interruptibility-state will be updated if
7064 * necessary. This cannot not be NULL.
7065 *
7066 * @remarks Requires CR0!
7067 * @remarks No-long-jump zone!!!
7068 */
7069static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7070 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7071{
7072 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7073 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7074 Assert(puIntrState);
7075 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7076
7077 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7078 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7079
7080#ifdef VBOX_STRICT
7081 /* Validate the error-code-valid bit for hardware exceptions. */
7082 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7083 {
7084 switch (uVector)
7085 {
7086 case X86_XCPT_PF:
7087 case X86_XCPT_DF:
7088 case X86_XCPT_TS:
7089 case X86_XCPT_NP:
7090 case X86_XCPT_SS:
7091 case X86_XCPT_GP:
7092 case X86_XCPT_AC:
7093 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7094 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7095 /* fallthru */
7096 default:
7097 break;
7098 }
7099 }
7100#endif
7101
7102 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7103 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7104 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7105
7106 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7107
7108 /* We require CR0 to check if the guest is in real-mode. */
7109 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7110 AssertRCReturn(rc, rc);
7111
7112 /*
7113 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7114 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7115 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7116 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7117 */
7118 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7119 {
7120 PVM pVM = pVCpu->CTX_SUFF(pVM);
7121 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7122 {
7123 Assert(PDMVmmDevHeapIsEnabled(pVM));
7124 Assert(pVM->hm.s.vmx.pRealModeTSS);
7125
7126 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7127 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7128 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7129 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7130 AssertRCReturn(rc, rc);
7131 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7132
7133 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7134 const size_t cbIdtEntry = sizeof(X86IDTR16);
7135 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7136 {
7137 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7138 if (uVector == X86_XCPT_DF)
7139 return VINF_EM_RESET;
7140 else if (uVector == X86_XCPT_GP)
7141 {
7142 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7143 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7144 }
7145
7146 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7147 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7148 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7149 }
7150
7151 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7152 uint16_t uGuestIp = pMixedCtx->ip;
7153 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7154 {
7155 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7156 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7157 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7158 }
7159 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7160 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7161
7162 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7163 X86IDTR16 IdtEntry;
7164 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7165 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7166 AssertRCReturn(rc, rc);
7167
7168 /* Construct the stack frame for the interrupt/exception handler. */
7169 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7170 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7171 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7172 AssertRCReturn(rc, rc);
7173
7174 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7175 if (rc == VINF_SUCCESS)
7176 {
7177 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7178 pMixedCtx->rip = IdtEntry.offSel;
7179 pMixedCtx->cs.Sel = IdtEntry.uSel;
7180 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7181 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7182 && uVector == X86_XCPT_PF)
7183 {
7184 pMixedCtx->cr2 = GCPtrFaultAddress;
7185 }
7186
7187 /* If any other guest-state bits are changed here, make sure to update
7188 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7189 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7190 | HM_CHANGED_GUEST_RIP
7191 | HM_CHANGED_GUEST_RFLAGS
7192 | HM_CHANGED_GUEST_RSP);
7193
7194 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7195 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7196 {
7197 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7198 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7199 Log4(("Clearing inhibition due to STI.\n"));
7200 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7201 }
7202 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7203
7204 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7205 it, if we are returning to ring-3 before executing guest code. */
7206 pVCpu->hm.s.Event.fPending = false;
7207 }
7208 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7209 return rc;
7210 }
7211 else
7212 {
7213 /*
7214 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7215 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7216 */
7217 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7218 }
7219 }
7220
7221 /* Validate. */
7222 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7223 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7224 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7225
7226 /* Inject. */
7227 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7228 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7229 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7230 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7231
7232 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7233 && uVector == X86_XCPT_PF)
7234 {
7235 pMixedCtx->cr2 = GCPtrFaultAddress;
7236 }
7237
7238 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7239 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7240
7241 AssertRCReturn(rc, rc);
7242 return rc;
7243}
7244
7245
7246/**
7247 * Clears the interrupt-window exiting control in the VMCS and if necessary
7248 * clears the current event in the VMCS as well.
7249 *
7250 * @returns VBox status code.
7251 * @param pVCpu Pointer to the VMCPU.
7252 *
7253 * @remarks Use this function only to clear events that have not yet been
7254 * delivered to the guest but are injected in the VMCS!
7255 * @remarks No-long-jump zone!!!
7256 */
7257static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7258{
7259 int rc;
7260 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7261
7262 /* Clear interrupt-window exiting control. */
7263 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7264 {
7265 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7266 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7267 AssertRC(rc);
7268 }
7269
7270 if (!pVCpu->hm.s.Event.fPending)
7271 return;
7272
7273#ifdef VBOX_STRICT
7274 uint32_t u32EntryInfo;
7275 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7276 AssertRC(rc);
7277 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7278#endif
7279
7280 /* Clear the entry-interruption field (including the valid bit). */
7281 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7282 AssertRC(rc);
7283
7284 /* Clear the pending debug exception field. */
7285 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7286 AssertRC(rc);
7287}
7288
7289
7290/**
7291 * Enters the VT-x session.
7292 *
7293 * @returns VBox status code.
7294 * @param pVM Pointer to the VM.
7295 * @param pVCpu Pointer to the VMCPU.
7296 * @param pCpu Pointer to the CPU info struct.
7297 */
7298VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7299{
7300 AssertPtr(pVM);
7301 AssertPtr(pVCpu);
7302 Assert(pVM->hm.s.vmx.fSupported);
7303 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7304 NOREF(pCpu);
7305
7306 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7307 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7308
7309#ifdef VBOX_STRICT
7310 /* Make sure we're in VMX root mode. */
7311 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7312 if (!(u32HostCR4 & X86_CR4_VMXE))
7313 {
7314 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7315 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7316 }
7317#endif
7318
7319 /*
7320 * Load the VCPU's VMCS as the current (and active) one.
7321 */
7322 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7323 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7324 if (RT_FAILURE(rc))
7325 return rc;
7326
7327 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7328 pVCpu->hm.s.fLeaveDone = false;
7329 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7330
7331 return VINF_SUCCESS;
7332}
7333
7334
7335/**
7336 * The thread-context callback (only on platforms which support it).
7337 *
7338 * @param enmEvent The thread-context event.
7339 * @param pVCpu Pointer to the VMCPU.
7340 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7341 * @thread EMT(pVCpu)
7342 */
7343VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7344{
7345 switch (enmEvent)
7346 {
7347 case RTTHREADCTXEVENT_PREEMPTING:
7348 {
7349 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7350 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7351 VMCPU_ASSERT_EMT(pVCpu);
7352
7353 PVM pVM = pVCpu->CTX_SUFF(pVM);
7354 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7355
7356 /* No longjmps (logger flushes, locks) in this fragile context. */
7357 VMMRZCallRing3Disable(pVCpu);
7358 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7359
7360 /*
7361 * Restore host-state (FPU, debug etc.)
7362 */
7363 if (!pVCpu->hm.s.fLeaveDone)
7364 {
7365 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7366 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7367 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7368 pVCpu->hm.s.fLeaveDone = true;
7369 }
7370
7371 /* Leave HM context, takes care of local init (term). */
7372 int rc = HMR0LeaveCpu(pVCpu);
7373 AssertRC(rc); NOREF(rc);
7374
7375 /* Restore longjmp state. */
7376 VMMRZCallRing3Enable(pVCpu);
7377 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7378 break;
7379 }
7380
7381 case RTTHREADCTXEVENT_RESUMED:
7382 {
7383 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7384 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7385 VMCPU_ASSERT_EMT(pVCpu);
7386
7387 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7388 VMMRZCallRing3Disable(pVCpu);
7389 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7390
7391 /* Initialize the bare minimum state required for HM. This takes care of
7392 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7393 int rc = HMR0EnterCpu(pVCpu);
7394 AssertRC(rc);
7395 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7396
7397 /* Load the active VMCS as the current one. */
7398 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7399 {
7400 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7401 AssertRC(rc); NOREF(rc);
7402 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7403 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7404 }
7405 pVCpu->hm.s.fLeaveDone = false;
7406
7407 /* Restore longjmp state. */
7408 VMMRZCallRing3Enable(pVCpu);
7409 break;
7410 }
7411
7412 default:
7413 break;
7414 }
7415}
7416
7417
7418/**
7419 * Saves the host state in the VMCS host-state.
7420 * Sets up the VM-exit MSR-load area.
7421 *
7422 * The CPU state will be loaded from these fields on every successful VM-exit.
7423 *
7424 * @returns VBox status code.
7425 * @param pVM Pointer to the VM.
7426 * @param pVCpu Pointer to the VMCPU.
7427 *
7428 * @remarks No-long-jump zone!!!
7429 */
7430static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7431{
7432 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7433
7434 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7435 return VINF_SUCCESS;
7436
7437 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7438 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7439
7440 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7441 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7442
7443 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7444 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7445
7446 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7447 return rc;
7448}
7449
7450
7451/**
7452 * Saves the host state in the VMCS host-state.
7453 *
7454 * @returns VBox status code.
7455 * @param pVM Pointer to the VM.
7456 * @param pVCpu Pointer to the VMCPU.
7457 *
7458 * @remarks No-long-jump zone!!!
7459 */
7460VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7461{
7462 AssertPtr(pVM);
7463 AssertPtr(pVCpu);
7464
7465 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7466
7467 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7468 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7469 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7470 return hmR0VmxSaveHostState(pVM, pVCpu);
7471}
7472
7473
7474/**
7475 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7476 * loaded from these fields on every successful VM-entry.
7477 *
7478 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7479 * Sets up the VM-entry controls.
7480 * Sets up the appropriate VMX non-root function to execute guest code based on
7481 * the guest CPU mode.
7482 *
7483 * @returns VBox status code.
7484 * @param pVM Pointer to the VM.
7485 * @param pVCpu Pointer to the VMCPU.
7486 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7487 * out-of-sync. Make sure to update the required fields
7488 * before using them.
7489 *
7490 * @remarks No-long-jump zone!!!
7491 */
7492static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7493{
7494 AssertPtr(pVM);
7495 AssertPtr(pVCpu);
7496 AssertPtr(pMixedCtx);
7497 HMVMX_ASSERT_PREEMPT_SAFE();
7498
7499#ifdef LOG_ENABLED
7500 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7501 * probably not initialized yet? Anyway this will do for now.
7502 *
7503 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7504 * interface and disable ring-3 calls when thread-context hooks are not
7505 * available. */
7506 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7507 VMMR0LogFlushDisable(pVCpu);
7508#endif
7509
7510 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7511
7512 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7513
7514 /* Determine real-on-v86 mode. */
7515 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7516 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7517 && CPUMIsGuestInRealModeEx(pMixedCtx))
7518 {
7519 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7520 }
7521
7522 /*
7523 * Load the guest-state into the VMCS.
7524 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7525 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7526 */
7527 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7528 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7529
7530 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7531 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7532 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7533
7534 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7535 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7536 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7537
7538 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7539 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7540
7541 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7542 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7543
7544 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7545 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7546 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7547
7548 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7549 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7550
7551 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7552 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7553
7554 /*
7555 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7556 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7557 */
7558 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7559 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7560
7561 /* Clear any unused and reserved bits. */
7562 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7563
7564#ifdef LOG_ENABLED
7565 /* Only reenable log-flushing if the caller has it enabled. */
7566 if (!fCallerDisabledLogFlush)
7567 VMMR0LogFlushEnable(pVCpu);
7568#endif
7569
7570 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7571 return rc;
7572}
7573
7574
7575/**
7576 * Loads the state shared between the host and guest into the VMCS.
7577 *
7578 * @param pVM Pointer to the VM.
7579 * @param pVCpu Pointer to the VMCPU.
7580 * @param pCtx Pointer to the guest-CPU context.
7581 *
7582 * @remarks No-long-jump zone!!!
7583 */
7584static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7585{
7586 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7587 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7588
7589 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7590 {
7591 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7592 AssertRC(rc);
7593 }
7594
7595 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7596 {
7597 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7598 AssertRC(rc);
7599
7600 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7601 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7602 {
7603 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7604 AssertRC(rc);
7605 }
7606 }
7607
7608 AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7609 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7610}
7611
7612
7613/**
7614 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7615 *
7616 * @param pVM Pointer to the VM.
7617 * @param pVCpu Pointer to the VMCPU.
7618 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7619 * out-of-sync. Make sure to update the required fields
7620 * before using them.
7621 */
7622DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7623{
7624 HMVMX_ASSERT_PREEMPT_SAFE();
7625
7626 Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7627#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7628 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7629#endif
7630
7631 if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7632 {
7633 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7634 AssertRC(rc);
7635 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7636 }
7637 else if (VMCPU_HMCF_VALUE(pVCpu))
7638 {
7639 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7640 AssertRC(rc);
7641 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7642 }
7643
7644 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7645 AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7646 || VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
7647 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7648
7649#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7650 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7651 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7652 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7653#endif
7654}
7655
7656
7657/**
7658 * Does the preparations before executing guest code in VT-x.
7659 *
7660 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7661 * recompiler. We must be cautious what we do here regarding committing
7662 * guest-state information into the VMCS assuming we assuredly execute the
7663 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7664 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7665 * so that the recompiler can (and should) use them when it resumes guest
7666 * execution. Otherwise such operations must be done when we can no longer
7667 * exit to ring-3.
7668 *
7669 * @returns Strict VBox status code.
7670 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7671 * have been disabled.
7672 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7673 * double-fault into the guest.
7674 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7675 *
7676 * @param pVM Pointer to the VM.
7677 * @param pVCpu Pointer to the VMCPU.
7678 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7679 * out-of-sync. Make sure to update the required fields
7680 * before using them.
7681 * @param pVmxTransient Pointer to the VMX transient structure.
7682 *
7683 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7684 * interrupts will be disabled.
7685 */
7686static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7687{
7688 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7689
7690#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7691 PGMRZDynMapFlushAutoSet(pVCpu);
7692#endif
7693
7694 /* Check force flag actions that might require us to go back to ring-3. */
7695 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7696 if (rc != VINF_SUCCESS)
7697 return rc;
7698
7699#ifndef IEM_VERIFICATION_MODE_FULL
7700 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7701 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7702 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7703 {
7704 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7705 RTGCPHYS GCPhysApicBase;
7706 GCPhysApicBase = pMixedCtx->msrApicBase;
7707 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7708
7709 /* Unalias any existing mapping. */
7710 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7711 AssertRCReturn(rc, rc);
7712
7713 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7714 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7715 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7716 AssertRCReturn(rc, rc);
7717
7718 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7719 }
7720#endif /* !IEM_VERIFICATION_MODE_FULL */
7721
7722 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7723 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7724
7725 /*
7726 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7727 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7728 */
7729 if (TRPMHasTrap(pVCpu))
7730 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7731 else if (!pVCpu->hm.s.Event.fPending)
7732 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7733
7734 /*
7735 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7736 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7737 */
7738 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7739 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7740 {
7741 Assert(rc == VINF_EM_RESET);
7742 return rc;
7743 }
7744
7745 /*
7746 * No longjmps to ring-3 from this point on!!!
7747 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7748 * This also disables flushing of the R0-logger instance (if any).
7749 */
7750 VMMRZCallRing3Disable(pVCpu);
7751
7752 /*
7753 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7754 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7755 *
7756 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7757 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7758 *
7759 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7760 * executing guest code.
7761 */
7762 pVmxTransient->uEflags = ASMIntDisableFlags();
7763 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7764 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7765 {
7766 hmR0VmxClearEventVmcs(pVCpu);
7767 ASMSetFlags(pVmxTransient->uEflags);
7768 VMMRZCallRing3Enable(pVCpu);
7769 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7770 return VINF_EM_RAW_TO_R3;
7771 }
7772 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7773 {
7774 hmR0VmxClearEventVmcs(pVCpu);
7775 ASMSetFlags(pVmxTransient->uEflags);
7776 VMMRZCallRing3Enable(pVCpu);
7777 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7778 return VINF_EM_RAW_INTERRUPT;
7779 }
7780
7781 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7782 pVCpu->hm.s.Event.fPending = false;
7783
7784 return VINF_SUCCESS;
7785}
7786
7787
7788/**
7789 * Prepares to run guest code in VT-x and we've committed to doing so. This
7790 * means there is no backing out to ring-3 or anywhere else at this
7791 * point.
7792 *
7793 * @param pVM Pointer to the VM.
7794 * @param pVCpu Pointer to the VMCPU.
7795 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7796 * out-of-sync. Make sure to update the required fields
7797 * before using them.
7798 * @param pVmxTransient Pointer to the VMX transient structure.
7799 *
7800 * @remarks Called with preemption disabled.
7801 * @remarks No-long-jump zone!!!
7802 */
7803static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7804{
7805 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7806 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7807 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7808
7809 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7810 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7811
7812 /*
7813 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7814 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7815 * Reload only the necessary state, the assertion will catch if other parts of the code
7816 * change.
7817 */
7818 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7819 {
7820 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7821 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7822 }
7823
7824#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7825 if (!CPUMIsGuestFPUStateActive(pVCpu))
7826 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7827 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7828#endif
7829
7830 /*
7831 * Load the host state bits as we may've been preempted (only happens when
7832 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7833 */
7834 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7835 {
7836 /* This ASSUMES that pfnStartVM has been set up already. */
7837 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7838 AssertRC(rc);
7839 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7840 }
7841 Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7842
7843 /*
7844 * Load the state shared between host and guest (FPU, debug).
7845 */
7846 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7847 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7848 AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7849
7850 /* Store status of the shared guest-host state at the time of VM-entry. */
7851#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7852 if (CPUMIsGuestInLongModeEx(pMixedCtx))
7853 {
7854 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7855 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7856 }
7857 else
7858#endif
7859 {
7860 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7861 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7862 }
7863 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7864
7865 /*
7866 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7867 */
7868 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7869 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7870
7871 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7872 RTCPUID idCurrentCpu = pCpu->idCpu;
7873 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7874 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7875 {
7876 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7877 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7878 }
7879
7880 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7881 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7882 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7883 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7884
7885 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7886
7887 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7888 to start executing. */
7889
7890#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7891 /*
7892 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7893 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7894 */
7895 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7896 {
7897 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7898 uint64_t u64HostTscAux = 0;
7899 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7900 AssertRC(rc2);
7901 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7902 }
7903#endif
7904}
7905
7906
7907/**
7908 * Performs some essential restoration of state after running guest code in
7909 * VT-x.
7910 *
7911 * @param pVM Pointer to the VM.
7912 * @param pVCpu Pointer to the VMCPU.
7913 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7914 * out-of-sync. Make sure to update the required fields
7915 * before using them.
7916 * @param pVmxTransient Pointer to the VMX transient structure.
7917 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7918 *
7919 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7920 *
7921 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7922 * unconditionally when it is safe to do so.
7923 */
7924static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7925{
7926 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7927
7928 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7929 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7930 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7931 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7932 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7933
7934 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7935 {
7936#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7937 /* Restore host's TSC_AUX. */
7938 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7939 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7940#endif
7941 /** @todo Find a way to fix hardcoding a guestimate. */
7942 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7943 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7944 }
7945
7946 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7947 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7948 Assert(!(ASMGetFlags() & X86_EFL_IF));
7949 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7950
7951#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7952 if (CPUMIsGuestFPUStateActive(pVCpu))
7953 {
7954 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7955 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7956 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7957 }
7958#endif
7959
7960 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7961 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7962 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7963 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7964
7965 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7966 uint32_t uExitReason;
7967 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7968 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7969 AssertRC(rc);
7970 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7971 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntInfo);
7972
7973 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7974 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7975 {
7976 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7977 pVmxTransient->fVMEntryFailed));
7978 return;
7979 }
7980
7981 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7982 {
7983 /* Update the guest interruptibility-state from the VMCS. */
7984 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7985#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7986 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7987 AssertRC(rc);
7988#endif
7989 /*
7990 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7991 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7992 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7993 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7994 */
7995 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7996 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7997 {
7998 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7999 AssertRC(rc);
8000 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8001 }
8002 }
8003}
8004
8005
8006
8007/**
8008 * Runs the guest code using VT-x the normal way.
8009 *
8010 * @returns VBox status code.
8011 * @param pVM Pointer to the VM.
8012 * @param pVCpu Pointer to the VMCPU.
8013 * @param pCtx Pointer to the guest-CPU context.
8014 *
8015 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
8016 * @remarks Called with preemption disabled.
8017 */
8018static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8019{
8020 VMXTRANSIENT VmxTransient;
8021 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8022 int rc = VERR_INTERNAL_ERROR_5;
8023 uint32_t cLoops = 0;
8024
8025 for (;; cLoops++)
8026 {
8027 Assert(!HMR0SuspendPending());
8028 HMVMX_ASSERT_CPU_SAFE();
8029
8030 /* Preparatory work for running guest code, this may force us to return
8031 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8032 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8033 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8034 if (rc != VINF_SUCCESS)
8035 break;
8036
8037 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8038 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8039 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8040
8041 /* Restore any residual host-state and save any bits shared between host
8042 and guest into the guest-CPU state. Re-enables interrupts! */
8043 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8044
8045 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8046 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8047 {
8048 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8049 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8050 return rc;
8051 }
8052
8053 /* Handle the VM-exit. */
8054 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8055 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8056 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8057 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8058 HMVMX_START_EXIT_DISPATCH_PROF();
8059#ifdef HMVMX_USE_FUNCTION_TABLE
8060 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8061#else
8062 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8063#endif
8064 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8065 if (rc != VINF_SUCCESS)
8066 break;
8067 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8068 {
8069 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8070 rc = VINF_EM_RAW_INTERRUPT;
8071 break;
8072 }
8073 }
8074
8075 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8076 return rc;
8077}
8078
8079
8080/**
8081 * Single steps guest code using VT-x.
8082 *
8083 * @returns VBox status code.
8084 * @param pVM Pointer to the VM.
8085 * @param pVCpu Pointer to the VMCPU.
8086 * @param pCtx Pointer to the guest-CPU context.
8087 *
8088 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8089 * @remarks Called with preemption disabled.
8090 */
8091static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8092{
8093 VMXTRANSIENT VmxTransient;
8094 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8095 int rc = VERR_INTERNAL_ERROR_5;
8096 uint32_t cLoops = 0;
8097 uint16_t uCsStart = pCtx->cs.Sel;
8098 uint64_t uRipStart = pCtx->rip;
8099
8100 for (;; cLoops++)
8101 {
8102 Assert(!HMR0SuspendPending());
8103 HMVMX_ASSERT_CPU_SAFE();
8104
8105 /* Preparatory work for running guest code, this may force us to return
8106 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8107 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8108 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8109 if (rc != VINF_SUCCESS)
8110 break;
8111
8112 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8113 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8114 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8115
8116 /* Restore any residual host-state and save any bits shared between host
8117 and guest into the guest-CPU state. Re-enables interrupts! */
8118 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8119
8120 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8121 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8122 {
8123 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8124 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8125 return rc;
8126 }
8127
8128 /* Handle the VM-exit. */
8129 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8130 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8131 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8132 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8133 HMVMX_START_EXIT_DISPATCH_PROF();
8134#ifdef HMVMX_USE_FUNCTION_TABLE
8135 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8136#else
8137 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8138#endif
8139 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8140 if (rc != VINF_SUCCESS)
8141 break;
8142 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8143 {
8144 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8145 rc = VINF_EM_RAW_INTERRUPT;
8146 break;
8147 }
8148
8149 /*
8150 * Did the RIP change, if so, consider it a single step.
8151 * Otherwise, make sure one of the TFs gets set.
8152 */
8153 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8154 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8155 AssertRCReturn(rc2, rc2);
8156 if ( pCtx->rip != uRipStart
8157 || pCtx->cs.Sel != uCsStart)
8158 {
8159 rc = VINF_EM_DBG_STEPPED;
8160 break;
8161 }
8162 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8163 }
8164
8165 /*
8166 * Clear the X86_EFL_TF if necessary.
8167 */
8168 if (pVCpu->hm.s.fClearTrapFlag)
8169 {
8170 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8171 AssertRCReturn(rc2, rc2);
8172 pVCpu->hm.s.fClearTrapFlag = false;
8173 pCtx->eflags.Bits.u1TF = 0;
8174 }
8175 /** @todo there seems to be issues with the resume flag when the monitor trap
8176 * flag is pending without being used. Seen early in bios init when
8177 * accessing APIC page in prot mode. */
8178
8179 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8180 return rc;
8181}
8182
8183
8184/**
8185 * Runs the guest code using VT-x.
8186 *
8187 * @returns VBox status code.
8188 * @param pVM Pointer to the VM.
8189 * @param pVCpu Pointer to the VMCPU.
8190 * @param pCtx Pointer to the guest-CPU context.
8191 *
8192 * @remarks Called with preemption disabled.
8193 */
8194VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8195{
8196 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8197 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8198 HMVMX_ASSERT_PREEMPT_SAFE();
8199
8200 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8201
8202 int rc;
8203 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8204 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8205 else
8206 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8207
8208 if (rc == VERR_EM_INTERPRETER)
8209 rc = VINF_EM_RAW_EMULATE_INSTR;
8210 else if (rc == VINF_EM_RESET)
8211 rc = VINF_EM_TRIPLE_FAULT;
8212
8213 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8214 if (RT_FAILURE(rc2))
8215 {
8216 pVCpu->hm.s.u32HMError = rc;
8217 rc = rc2;
8218 }
8219 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8220 return rc;
8221}
8222
8223
8224#ifndef HMVMX_USE_FUNCTION_TABLE
8225DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8226{
8227 int rc;
8228 switch (rcReason)
8229 {
8230 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8231 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8232 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8233 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8234 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8235 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8236 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8237 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8238 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8239 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8240 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8241 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8242 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8243 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8244 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8245 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8246 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8247 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8248 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8249 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8250 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8251 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8252 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8253 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8254 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8255 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8256 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8257 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8258 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8259 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8260 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8261 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8262 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8263
8264 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8265 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8266 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8267 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8268 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8269 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8270 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8271 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8272 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8273
8274 case VMX_EXIT_VMCALL:
8275 case VMX_EXIT_VMCLEAR:
8276 case VMX_EXIT_VMLAUNCH:
8277 case VMX_EXIT_VMPTRLD:
8278 case VMX_EXIT_VMPTRST:
8279 case VMX_EXIT_VMREAD:
8280 case VMX_EXIT_VMRESUME:
8281 case VMX_EXIT_VMWRITE:
8282 case VMX_EXIT_VMXOFF:
8283 case VMX_EXIT_VMXON:
8284 case VMX_EXIT_INVEPT:
8285 case VMX_EXIT_INVVPID:
8286 case VMX_EXIT_VMFUNC:
8287 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8288 break;
8289 default:
8290 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8291 break;
8292 }
8293 return rc;
8294}
8295#endif
8296
8297#ifdef DEBUG
8298/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8299# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8300 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8301
8302# define HMVMX_ASSERT_PREEMPT_CPUID() \
8303 do \
8304 { \
8305 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8306 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8307 } while (0)
8308
8309# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8310 do { \
8311 AssertPtr(pVCpu); \
8312 AssertPtr(pMixedCtx); \
8313 AssertPtr(pVmxTransient); \
8314 Assert(pVmxTransient->fVMEntryFailed == false); \
8315 Assert(ASMIntAreEnabled()); \
8316 HMVMX_ASSERT_PREEMPT_SAFE(); \
8317 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8318 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8319 HMVMX_ASSERT_PREEMPT_SAFE(); \
8320 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8321 HMVMX_ASSERT_PREEMPT_CPUID(); \
8322 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8323 } while (0)
8324
8325# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8326 do { \
8327 Log4Func(("\n")); \
8328 } while(0)
8329#else /* Release builds */
8330# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8331# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8332#endif
8333
8334
8335/**
8336 * Advances the guest RIP after reading it from the VMCS.
8337 *
8338 * @returns VBox status code.
8339 * @param pVCpu Pointer to the VMCPU.
8340 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8341 * out-of-sync. Make sure to update the required fields
8342 * before using them.
8343 * @param pVmxTransient Pointer to the VMX transient structure.
8344 *
8345 * @remarks No-long-jump zone!!!
8346 */
8347DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8348{
8349 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8350 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8351 AssertRCReturn(rc, rc);
8352
8353 pMixedCtx->rip += pVmxTransient->cbInstr;
8354 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8355 return rc;
8356}
8357
8358
8359/**
8360 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8361 * and update error record fields accordingly.
8362 *
8363 * @return VMX_IGS_* return codes.
8364 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8365 * wrong with the guest state.
8366 *
8367 * @param pVM Pointer to the VM.
8368 * @param pVCpu Pointer to the VMCPU.
8369 * @param pCtx Pointer to the guest-CPU state.
8370 */
8371static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8372{
8373#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8374#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8375 uError = (err); \
8376 break; \
8377 } else do {} while (0)
8378/* Duplicate of IEM_IS_CANONICAL(). */
8379#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8380
8381 int rc;
8382 uint32_t uError = VMX_IGS_ERROR;
8383 uint32_t u32Val;
8384 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8385
8386 do
8387 {
8388 /*
8389 * CR0.
8390 */
8391 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8392 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8393 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8394 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8395 if (fUnrestrictedGuest)
8396 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8397
8398 uint32_t u32GuestCR0;
8399 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8400 AssertRCBreak(rc);
8401 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8402 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8403 if ( !fUnrestrictedGuest
8404 && (u32GuestCR0 & X86_CR0_PG)
8405 && !(u32GuestCR0 & X86_CR0_PE))
8406 {
8407 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8408 }
8409
8410 /*
8411 * CR4.
8412 */
8413 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8414 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8415
8416 uint32_t u32GuestCR4;
8417 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8418 AssertRCBreak(rc);
8419 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8420 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8421
8422 /*
8423 * IA32_DEBUGCTL MSR.
8424 */
8425 uint64_t u64Val;
8426 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8427 AssertRCBreak(rc);
8428 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8429 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8430 {
8431 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8432 }
8433 uint64_t u64DebugCtlMsr = u64Val;
8434
8435#ifdef VBOX_STRICT
8436 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8437 AssertRCBreak(rc);
8438 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8439#endif
8440 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8441
8442 /*
8443 * RIP and RFLAGS.
8444 */
8445 uint32_t u32Eflags;
8446#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8447 if (HMVMX_IS_64BIT_HOST_MODE())
8448 {
8449 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8450 AssertRCBreak(rc);
8451 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8452 if ( !fLongModeGuest
8453 || !pCtx->cs.Attr.n.u1Long)
8454 {
8455 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8456 }
8457 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8458 * must be identical if the "IA32e mode guest" VM-entry control is 1
8459 * and CS.L is 1. No check applies if the CPU supports 64
8460 * linear-address bits. */
8461
8462 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8463 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8464 AssertRCBreak(rc);
8465 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8466 VMX_IGS_RFLAGS_RESERVED);
8467 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8468 u32Eflags = u64Val;
8469 }
8470 else
8471#endif
8472 {
8473 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8474 AssertRCBreak(rc);
8475 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8476 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8477 }
8478
8479 if ( fLongModeGuest
8480 || ( fUnrestrictedGuest
8481 && !(u32GuestCR0 & X86_CR0_PE)))
8482 {
8483 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8484 }
8485
8486 uint32_t u32EntryInfo;
8487 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8488 AssertRCBreak(rc);
8489 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8490 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8491 {
8492 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8493 }
8494
8495 /*
8496 * 64-bit checks.
8497 */
8498#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8499 if (HMVMX_IS_64BIT_HOST_MODE())
8500 {
8501 if ( fLongModeGuest
8502 && !fUnrestrictedGuest)
8503 {
8504 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8505 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8506 }
8507
8508 if ( !fLongModeGuest
8509 && (u32GuestCR4 & X86_CR4_PCIDE))
8510 {
8511 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8512 }
8513
8514 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8515 * 51:32 beyond the processor's physical-address width are 0. */
8516
8517 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8518 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8519 {
8520 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8521 }
8522
8523 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8524 AssertRCBreak(rc);
8525 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8526
8527 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8528 AssertRCBreak(rc);
8529 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8530 }
8531#endif
8532
8533 /*
8534 * PERF_GLOBAL MSR.
8535 */
8536 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8537 {
8538 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8539 AssertRCBreak(rc);
8540 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8541 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8542 }
8543
8544 /*
8545 * PAT MSR.
8546 */
8547 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8548 {
8549 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8550 AssertRCBreak(rc);
8551 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8552 for (unsigned i = 0; i < 8; i++)
8553 {
8554 uint8_t u8Val = (u64Val & 0x7);
8555 if ( u8Val != 0 /* UC */
8556 || u8Val != 1 /* WC */
8557 || u8Val != 4 /* WT */
8558 || u8Val != 5 /* WP */
8559 || u8Val != 6 /* WB */
8560 || u8Val != 7 /* UC- */)
8561 {
8562 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8563 }
8564 u64Val >>= 3;
8565 }
8566 }
8567
8568 /*
8569 * EFER MSR.
8570 */
8571 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8572 {
8573 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8574 AssertRCBreak(rc);
8575 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8576 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8577 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8578 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8579 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8580 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8581 }
8582
8583 /*
8584 * Segment registers.
8585 */
8586 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8587 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8588 if (!(u32Eflags & X86_EFL_VM))
8589 {
8590 /* CS */
8591 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8592 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8593 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8594 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8595 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8596 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8597 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8598 /* CS cannot be loaded with NULL in protected mode. */
8599 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8600 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8601 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8602 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8603 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8604 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8605 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8606 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8607 else
8608 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8609
8610 /* SS */
8611 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8612 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8613 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8614 if ( !(pCtx->cr0 & X86_CR0_PE)
8615 || pCtx->cs.Attr.n.u4Type == 3)
8616 {
8617 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8618 }
8619 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8620 {
8621 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8622 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8623 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8624 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8625 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8626 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8627 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8628 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8629 }
8630
8631 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8632 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8633 {
8634 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8635 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8636 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8637 || pCtx->ds.Attr.n.u4Type > 11
8638 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8639 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8640 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8641 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8642 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8643 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8644 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8645 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8646 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8647 }
8648 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8649 {
8650 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8651 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8652 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8653 || pCtx->es.Attr.n.u4Type > 11
8654 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8655 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8656 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8657 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8658 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8659 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8660 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8661 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8662 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8663 }
8664 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8665 {
8666 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8667 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8668 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8669 || pCtx->fs.Attr.n.u4Type > 11
8670 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8671 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8672 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8673 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8674 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8675 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8676 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8677 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8678 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8679 }
8680 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8681 {
8682 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8683 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8684 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8685 || pCtx->gs.Attr.n.u4Type > 11
8686 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8687 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8688 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8689 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8690 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8691 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8692 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8693 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8694 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8695 }
8696 /* 64-bit capable CPUs. */
8697#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8698 if (HMVMX_IS_64BIT_HOST_MODE())
8699 {
8700 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8701 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8702 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8703 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8704 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8705 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8706 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8707 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8708 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8709 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8710 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8711 }
8712#endif
8713 }
8714 else
8715 {
8716 /* V86 mode checks. */
8717 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8718 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8719 {
8720 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8721 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8722 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8723 }
8724 else
8725 {
8726 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8727 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8728 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8729 }
8730
8731 /* CS */
8732 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8733 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8734 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8735 /* SS */
8736 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8737 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8738 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8739 /* DS */
8740 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8741 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8742 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8743 /* ES */
8744 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8745 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8746 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8747 /* FS */
8748 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8749 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8750 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8751 /* GS */
8752 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8753 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8754 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8755 /* 64-bit capable CPUs. */
8756#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8757 if (HMVMX_IS_64BIT_HOST_MODE())
8758 {
8759 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8760 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8761 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8762 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8763 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8764 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8765 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8766 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8767 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8768 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8769 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8770 }
8771#endif
8772 }
8773
8774 /*
8775 * TR.
8776 */
8777 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8778 /* 64-bit capable CPUs. */
8779#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8780 if (HMVMX_IS_64BIT_HOST_MODE())
8781 {
8782 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8783 }
8784#endif
8785 if (fLongModeGuest)
8786 {
8787 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8788 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8789 }
8790 else
8791 {
8792 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8793 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8794 VMX_IGS_TR_ATTR_TYPE_INVALID);
8795 }
8796 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8797 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8798 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8799 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8800 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8801 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8802 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8803 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8804
8805 /*
8806 * GDTR and IDTR.
8807 */
8808#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8809 if (HMVMX_IS_64BIT_HOST_MODE())
8810 {
8811 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8812 AssertRCBreak(rc);
8813 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8814
8815 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8816 AssertRCBreak(rc);
8817 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8818 }
8819#endif
8820
8821 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8822 AssertRCBreak(rc);
8823 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8824
8825 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8826 AssertRCBreak(rc);
8827 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8828
8829 /*
8830 * Guest Non-Register State.
8831 */
8832 /* Activity State. */
8833 uint32_t u32ActivityState;
8834 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8835 AssertRCBreak(rc);
8836 HMVMX_CHECK_BREAK( !u32ActivityState
8837 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8838 VMX_IGS_ACTIVITY_STATE_INVALID);
8839 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8840 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8841 uint32_t u32IntrState;
8842 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8843 AssertRCBreak(rc);
8844 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8845 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8846 {
8847 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8848 }
8849
8850 /** @todo Activity state and injecting interrupts. Left as a todo since we
8851 * currently don't use activity states but ACTIVE. */
8852
8853 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8854 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8855
8856 /* Guest interruptibility-state. */
8857 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8858 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8859 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8860 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8861 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8862 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8863 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8864 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8865 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8866 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8867 {
8868 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8869 {
8870 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8871 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8872 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8873 }
8874 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8875 {
8876 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8877 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8878 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8879 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8880 }
8881 }
8882 /** @todo Assumes the processor is not in SMM. */
8883 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8884 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8885 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8886 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8887 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8888 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8889 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8890 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8891 {
8892 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8893 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8894 }
8895
8896 /* Pending debug exceptions. */
8897 if (HMVMX_IS_64BIT_HOST_MODE())
8898 {
8899 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8900 AssertRCBreak(rc);
8901 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8902 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8903 u32Val = u64Val; /* For pending debug exceptions checks below. */
8904 }
8905 else
8906 {
8907 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8908 AssertRCBreak(rc);
8909 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8910 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8911 }
8912
8913 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8914 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8915 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8916 {
8917 if ( (u32Eflags & X86_EFL_TF)
8918 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8919 {
8920 /* Bit 14 is PendingDebug.BS. */
8921 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8922 }
8923 if ( !(u32Eflags & X86_EFL_TF)
8924 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8925 {
8926 /* Bit 14 is PendingDebug.BS. */
8927 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8928 }
8929 }
8930
8931 /* VMCS link pointer. */
8932 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8933 AssertRCBreak(rc);
8934 if (u64Val != UINT64_C(0xffffffffffffffff))
8935 {
8936 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8937 /** @todo Bits beyond the processor's physical-address width MBZ. */
8938 /** @todo 32-bit located in memory referenced by value of this field (as a
8939 * physical address) must contain the processor's VMCS revision ID. */
8940 /** @todo SMM checks. */
8941 }
8942
8943 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8944
8945 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8946 if (uError == VMX_IGS_ERROR)
8947 uError = VMX_IGS_REASON_NOT_FOUND;
8948 } while (0);
8949
8950 pVCpu->hm.s.u32HMError = uError;
8951 return uError;
8952
8953#undef HMVMX_ERROR_BREAK
8954#undef HMVMX_CHECK_BREAK
8955#undef HMVMX_IS_CANONICAL
8956}
8957
8958/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8959/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8960/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8961
8962/** @name VM-exit handlers.
8963 * @{
8964 */
8965
8966/**
8967 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8968 */
8969HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8970{
8971 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8972 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8973 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8974#if HC_ARCH_BITS == 64
8975 Assert(ASMIntAreEnabled());
8976 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8977 return VINF_SUCCESS;
8978#endif
8979 return VINF_EM_RAW_INTERRUPT;
8980}
8981
8982
8983/**
8984 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8985 */
8986HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8987{
8988 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8989 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8990
8991 int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8992 AssertRCReturn(rc, rc);
8993
8994 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
8995 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8996 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8997 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
8998
8999 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9000 {
9001 /*
9002 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9003 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9004 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9005 *
9006 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9007 */
9008 VMXDispatchHostNmi();
9009 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9010 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9011 return VINF_SUCCESS;
9012 }
9013
9014 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9015 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9016 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9017 {
9018 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9019 return VINF_SUCCESS;
9020 }
9021 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9022 {
9023 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9024 return rc;
9025 }
9026
9027 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9028 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9029 switch (uIntType)
9030 {
9031 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9032 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
9033 /* no break */
9034 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9035 {
9036 switch (uVector)
9037 {
9038 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9039 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9040 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9041 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9042 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9043 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9044#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9045 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9046 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9047 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9048 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9049 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9050 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9051 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9052 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9053 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9054 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9055#endif
9056 default:
9057 {
9058 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9059 AssertRCReturn(rc, rc);
9060
9061 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9062 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9063 {
9064 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9065 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9066 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9067
9068 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9069 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
9070 AssertRCReturn(rc, rc);
9071 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9072 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9073 0 /* GCPtrFaultAddress */);
9074 AssertRCReturn(rc, rc);
9075 }
9076 else
9077 {
9078 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9079 pVCpu->hm.s.u32HMError = uVector;
9080 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9081 }
9082 break;
9083 }
9084 }
9085 break;
9086 }
9087
9088 default:
9089 {
9090 pVCpu->hm.s.u32HMError = uExitIntInfo;
9091 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9092 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9093 break;
9094 }
9095 }
9096 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9097 return rc;
9098}
9099
9100
9101/**
9102 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9103 */
9104HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9105{
9106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9107
9108 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9109 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9110 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9111 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9112 AssertRCReturn(rc, rc);
9113
9114 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9115 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9116 return VINF_SUCCESS;
9117}
9118
9119
9120/**
9121 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9122 */
9123HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9124{
9125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9126 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9127 HMVMX_RETURN_UNEXPECTED_EXIT();
9128}
9129
9130
9131/**
9132 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9133 */
9134HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9135{
9136 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9137 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9138 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9139}
9140
9141
9142/**
9143 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9144 */
9145HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9146{
9147 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9149 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9150}
9151
9152
9153/**
9154 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9155 */
9156HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9157{
9158 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9159 PVM pVM = pVCpu->CTX_SUFF(pVM);
9160 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9161 if (RT_LIKELY(rc == VINF_SUCCESS))
9162 {
9163 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9164 Assert(pVmxTransient->cbInstr == 2);
9165 }
9166 else
9167 {
9168 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9169 rc = VERR_EM_INTERPRETER;
9170 }
9171 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9172 return rc;
9173}
9174
9175
9176/**
9177 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9178 */
9179HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9180{
9181 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9182 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9183 AssertRCReturn(rc, rc);
9184
9185 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9186 return VINF_EM_RAW_EMULATE_INSTR;
9187
9188 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9189 HMVMX_RETURN_UNEXPECTED_EXIT();
9190}
9191
9192
9193/**
9194 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9195 */
9196HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9197{
9198 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9199 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9200 AssertRCReturn(rc, rc);
9201
9202 PVM pVM = pVCpu->CTX_SUFF(pVM);
9203 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9204 if (RT_LIKELY(rc == VINF_SUCCESS))
9205 {
9206 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9207 Assert(pVmxTransient->cbInstr == 2);
9208 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9209 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9210 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9211 }
9212 else
9213 {
9214 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9215 rc = VERR_EM_INTERPRETER;
9216 }
9217 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9218 return rc;
9219}
9220
9221
9222/**
9223 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9224 */
9225HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9226{
9227 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9228 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9229 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9230 AssertRCReturn(rc, rc);
9231
9232 PVM pVM = pVCpu->CTX_SUFF(pVM);
9233 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9234 if (RT_LIKELY(rc == VINF_SUCCESS))
9235 {
9236 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9237 Assert(pVmxTransient->cbInstr == 3);
9238 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9239 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9240 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9241 }
9242 else
9243 {
9244 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9245 rc = VERR_EM_INTERPRETER;
9246 }
9247 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9248 return rc;
9249}
9250
9251
9252/**
9253 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9254 */
9255HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9256{
9257 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9258 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9259 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9260 AssertRCReturn(rc, rc);
9261
9262 PVM pVM = pVCpu->CTX_SUFF(pVM);
9263 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9264 if (RT_LIKELY(rc == VINF_SUCCESS))
9265 {
9266 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9267 Assert(pVmxTransient->cbInstr == 2);
9268 }
9269 else
9270 {
9271 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9272 rc = VERR_EM_INTERPRETER;
9273 }
9274 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9275 return rc;
9276}
9277
9278
9279/**
9280 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9281 */
9282HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9283{
9284 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9285 PVM pVM = pVCpu->CTX_SUFF(pVM);
9286 Assert(!pVM->hm.s.fNestedPaging);
9287
9288 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9289 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9290 AssertRCReturn(rc, rc);
9291
9292 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9293 rc = VBOXSTRICTRC_VAL(rc2);
9294 if (RT_LIKELY(rc == VINF_SUCCESS))
9295 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9296 else
9297 {
9298 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9299 pVmxTransient->uExitQualification, rc));
9300 }
9301 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9302 return rc;
9303}
9304
9305
9306/**
9307 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9308 */
9309HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9310{
9311 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9312 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9313 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9314 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9315 AssertRCReturn(rc, rc);
9316
9317 PVM pVM = pVCpu->CTX_SUFF(pVM);
9318 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9319 if (RT_LIKELY(rc == VINF_SUCCESS))
9320 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9321 else
9322 {
9323 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9324 rc = VERR_EM_INTERPRETER;
9325 }
9326 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9327 return rc;
9328}
9329
9330
9331/**
9332 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9333 */
9334HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9335{
9336 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9337 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9338 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9339 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9340 AssertRCReturn(rc, rc);
9341
9342 PVM pVM = pVCpu->CTX_SUFF(pVM);
9343 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9344 rc = VBOXSTRICTRC_VAL(rc2);
9345 if (RT_LIKELY( rc == VINF_SUCCESS
9346 || rc == VINF_EM_HALT))
9347 {
9348 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9349 AssertRCReturn(rc3, rc3);
9350
9351 if ( rc == VINF_EM_HALT
9352 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9353 {
9354 rc = VINF_SUCCESS;
9355 }
9356 }
9357 else
9358 {
9359 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9360 rc = VERR_EM_INTERPRETER;
9361 }
9362 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9363 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9365 return rc;
9366}
9367
9368
9369/**
9370 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9371 */
9372HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9373{
9374 /*
9375 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9376 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9377 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9378 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9379 */
9380 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9381 HMVMX_RETURN_UNEXPECTED_EXIT();
9382}
9383
9384
9385/**
9386 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9387 */
9388HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9389{
9390 /*
9391 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9392 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9393 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9394 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9395 */
9396 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9397 HMVMX_RETURN_UNEXPECTED_EXIT();
9398}
9399
9400
9401/**
9402 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9403 */
9404HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9405{
9406 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9407 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9408 HMVMX_RETURN_UNEXPECTED_EXIT();
9409}
9410
9411
9412/**
9413 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9414 */
9415HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9416{
9417 /*
9418 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9419 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9420 * See Intel spec. 25.3 "Other Causes of VM-exits".
9421 */
9422 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9423 HMVMX_RETURN_UNEXPECTED_EXIT();
9424}
9425
9426
9427/**
9428 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9429 * VM-exit.
9430 */
9431HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9432{
9433 /*
9434 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9435 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9436 *
9437 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9438 * See Intel spec. "23.8 Restrictions on VMX operation".
9439 */
9440 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9441 return VINF_SUCCESS;
9442}
9443
9444
9445/**
9446 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9447 * VM-exit.
9448 */
9449HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9450{
9451 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9452 return VINF_EM_RESET;
9453}
9454
9455
9456/**
9457 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9458 */
9459HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9460{
9461 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9462 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9463 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9464 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9465 AssertRCReturn(rc, rc);
9466
9467 pMixedCtx->rip++;
9468 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9469 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9470 rc = VINF_SUCCESS;
9471 else
9472 rc = VINF_EM_HALT;
9473
9474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9475 return rc;
9476}
9477
9478
9479/**
9480 * VM-exit handler for instructions that result in a #UD exception delivered to
9481 * the guest.
9482 */
9483HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9484{
9485 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9486 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9487 return VINF_SUCCESS;
9488}
9489
9490
9491/**
9492 * VM-exit handler for expiry of the VMX preemption timer.
9493 */
9494HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9495{
9496 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9497
9498 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9499 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9500
9501 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9502 PVM pVM = pVCpu->CTX_SUFF(pVM);
9503 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9505 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9506}
9507
9508
9509/**
9510 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9511 */
9512HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9513{
9514 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9515
9516 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9517 /** @todo check if XSETBV is supported by the recompiler. */
9518 return VERR_EM_INTERPRETER;
9519}
9520
9521
9522/**
9523 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9524 */
9525HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9526{
9527 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9528
9529 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9530 /** @todo implement EMInterpretInvpcid() */
9531 return VERR_EM_INTERPRETER;
9532}
9533
9534
9535/**
9536 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9537 * Error VM-exit.
9538 */
9539HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9540{
9541 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9542 AssertRCReturn(rc, rc);
9543
9544 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9545 NOREF(uInvalidReason);
9546
9547#ifdef VBOX_STRICT
9548 uint32_t uIntrState;
9549 HMVMXHCUINTREG uHCReg;
9550 uint64_t u64Val;
9551 uint32_t u32Val;
9552
9553 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9554 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9555 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9556 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9557 AssertRCReturn(rc, rc);
9558
9559 Log4(("uInvalidReason %u\n", uInvalidReason));
9560 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9561 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9562 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9563 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9564
9565 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9566 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9567 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9568 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9569 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9570 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9571 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9572 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9573 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9574 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9575 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9576 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9577#endif
9578
9579 PVM pVM = pVCpu->CTX_SUFF(pVM);
9580 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9581
9582 return VERR_VMX_INVALID_GUEST_STATE;
9583}
9584
9585
9586/**
9587 * VM-exit handler for VM-entry failure due to an MSR-load
9588 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9589 */
9590HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9591{
9592 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9593 HMVMX_RETURN_UNEXPECTED_EXIT();
9594}
9595
9596
9597/**
9598 * VM-exit handler for VM-entry failure due to a machine-check event
9599 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9600 */
9601HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9602{
9603 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9604 HMVMX_RETURN_UNEXPECTED_EXIT();
9605}
9606
9607
9608/**
9609 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9610 * theory.
9611 */
9612HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9613{
9614 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9615 return VERR_VMX_UNDEFINED_EXIT_CODE;
9616}
9617
9618
9619/**
9620 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9621 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9622 * Conditional VM-exit.
9623 */
9624HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9625{
9626 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9627
9628 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9629 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9630 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9631 return VERR_EM_INTERPRETER;
9632 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9633 HMVMX_RETURN_UNEXPECTED_EXIT();
9634}
9635
9636
9637/**
9638 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9639 */
9640HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9641{
9642 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9643
9644 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9646 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9647 return VERR_EM_INTERPRETER;
9648 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9649 HMVMX_RETURN_UNEXPECTED_EXIT();
9650}
9651
9652
9653/**
9654 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9655 */
9656HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9657{
9658 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9659
9660 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9661 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9662 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9663 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9664 AssertRCReturn(rc, rc);
9665 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9666
9667 PVM pVM = pVCpu->CTX_SUFF(pVM);
9668 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9669 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9670 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9672
9673 if (RT_LIKELY(rc == VINF_SUCCESS))
9674 {
9675 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9676 Assert(pVmxTransient->cbInstr == 2);
9677 }
9678 return rc;
9679}
9680
9681
9682/**
9683 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9684 */
9685HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9686{
9687 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9688 PVM pVM = pVCpu->CTX_SUFF(pVM);
9689 int rc = VINF_SUCCESS;
9690
9691 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9692 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9693 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9694 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9695 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9696 AssertRCReturn(rc, rc);
9697 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9698
9699 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9700 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9701 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9702
9703 if (RT_LIKELY(rc == VINF_SUCCESS))
9704 {
9705 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9706
9707 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9708 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9709 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9710 {
9711 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9712 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9713 EMInterpretWrmsr() changes it. */
9714 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9715 }
9716 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9717 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9718 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9719 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9720
9721 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9722 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9723 {
9724 switch (pMixedCtx->ecx)
9725 {
9726 case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9727 case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9728 case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9729 case MSR_K8_FS_BASE: /* no break */
9730 case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9731 case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9732 }
9733 }
9734#ifdef VBOX_STRICT
9735 else
9736 {
9737 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9738 switch (pMixedCtx->ecx)
9739 {
9740 case MSR_IA32_SYSENTER_CS:
9741 case MSR_IA32_SYSENTER_EIP:
9742 case MSR_IA32_SYSENTER_ESP:
9743 case MSR_K8_FS_BASE:
9744 case MSR_K8_GS_BASE:
9745 {
9746 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9747 HMVMX_RETURN_UNEXPECTED_EXIT();
9748 }
9749
9750 case MSR_K8_LSTAR:
9751 case MSR_K6_STAR:
9752 case MSR_K8_SF_MASK:
9753 case MSR_K8_TSC_AUX:
9754 case MSR_K8_KERNEL_GS_BASE:
9755 {
9756 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9757 pMixedCtx->ecx));
9758 HMVMX_RETURN_UNEXPECTED_EXIT();
9759 }
9760 }
9761 }
9762#endif /* VBOX_STRICT */
9763 }
9764 return rc;
9765}
9766
9767
9768/**
9769 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9770 */
9771HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9772{
9773 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9774
9775 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9776 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9777 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9778 return VERR_EM_INTERPRETER;
9779 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9780 HMVMX_RETURN_UNEXPECTED_EXIT();
9781}
9782
9783
9784/**
9785 * VM-exit handler for when the TPR value is lowered below the specified
9786 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9787 */
9788HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9789{
9790 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9791 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9792
9793 /*
9794 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9795 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9796 * resume guest execution.
9797 */
9798 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9800 return VINF_SUCCESS;
9801}
9802
9803
9804/**
9805 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9806 * VM-exit.
9807 *
9808 * @retval VINF_SUCCESS when guest execution can continue.
9809 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9810 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9811 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9812 * recompiler.
9813 */
9814HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9815{
9816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9817 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9818 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9819 AssertRCReturn(rc, rc);
9820
9821 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9822 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9823 PVM pVM = pVCpu->CTX_SUFF(pVM);
9824 switch (uAccessType)
9825 {
9826 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9827 {
9828#if 0
9829 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9830 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9831#else
9832 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9833 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9834 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9835#endif
9836 AssertRCReturn(rc, rc);
9837
9838 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9839 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9840 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9841 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9842
9843 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9844 {
9845 case 0: /* CR0 */
9846 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9847 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9848 break;
9849 case 2: /* CR2 */
9850 /* Nothing to do here, CR2 it's not part of the VMCS. */
9851 break;
9852 case 3: /* CR3 */
9853 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9854 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9855 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9856 break;
9857 case 4: /* CR4 */
9858 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9859 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9860 break;
9861 case 8: /* CR8 */
9862 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9863 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9864 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9865 break;
9866 default:
9867 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9868 break;
9869 }
9870
9871 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9872 break;
9873 }
9874
9875 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9876 {
9877 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9878 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9879 AssertRCReturn(rc, rc);
9880 Assert( !pVM->hm.s.fNestedPaging
9881 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9882 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9883
9884 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9885 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9886 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9887
9888 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9889 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9890 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9891 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9892 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9893 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9894 break;
9895 }
9896
9897 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9898 {
9899 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9900 AssertRCReturn(rc, rc);
9901 rc = EMInterpretCLTS(pVM, pVCpu);
9902 AssertRCReturn(rc, rc);
9903 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9904 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9905 Log4(("CRX CLTS write rc=%d\n", rc));
9906 break;
9907 }
9908
9909 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9910 {
9911 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9912 AssertRCReturn(rc, rc);
9913 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9914 if (RT_LIKELY(rc == VINF_SUCCESS))
9915 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9917 Log4(("CRX LMSW write rc=%d\n", rc));
9918 break;
9919 }
9920
9921 default:
9922 {
9923 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9924 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9925 }
9926 }
9927
9928 /* Validate possible error codes. */
9929 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9930 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9931 if (RT_SUCCESS(rc))
9932 {
9933 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9934 AssertRCReturn(rc2, rc2);
9935 }
9936
9937 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9938 return rc;
9939}
9940
9941
9942/**
9943 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9944 * VM-exit.
9945 */
9946HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9947{
9948 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9949 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9950
9951 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9952 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9953 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9954 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9955 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9956 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9957 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9958 AssertRCReturn(rc2, rc2);
9959
9960 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9961 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9962 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9963 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9964 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9965 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9966 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9967
9968 /* I/O operation lookup arrays. */
9969 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9970 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9971
9972 VBOXSTRICTRC rcStrict;
9973 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9974 const uint32_t cbInstr = pVmxTransient->cbInstr;
9975 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9976 PVM pVM = pVCpu->CTX_SUFF(pVM);
9977 if (fIOString)
9978 {
9979#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9980 /*
9981 * INS/OUTS - I/O String instruction.
9982 *
9983 * Use instruction-information if available, otherwise fall back on
9984 * interpreting the instruction.
9985 */
9986 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9987 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9988 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9989 {
9990 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9991 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9992 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9993 AssertRCReturn(rc2, rc2);
9994 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9995 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9996 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9997 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9998 if (fIOWrite)
9999 {
10000 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10001 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10002 }
10003 else
10004 {
10005 /*
10006 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10007 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10008 * See Intel Instruction spec. for "INS".
10009 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10010 */
10011 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10012 }
10013 }
10014 else
10015 {
10016 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10017 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10018 AssertRCReturn(rc2, rc2);
10019 rcStrict = IEMExecOne(pVCpu);
10020 }
10021 /** @todo IEM needs to be setting these flags somehow. */
10022 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10023 fUpdateRipAlready = true;
10024#else
10025 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10026 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
10027 if (RT_SUCCESS(rcStrict))
10028 {
10029 if (fIOWrite)
10030 {
10031 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10032 (DISCPUMODE)pDis->uAddrMode, cbValue);
10033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10034 }
10035 else
10036 {
10037 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10038 (DISCPUMODE)pDis->uAddrMode, cbValue);
10039 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10040 }
10041 }
10042 else
10043 {
10044 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10045 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10046 }
10047#endif
10048 }
10049 else
10050 {
10051 /*
10052 * IN/OUT - I/O instruction.
10053 */
10054 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10055 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10056 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10057 if (fIOWrite)
10058 {
10059 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10060 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10061 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10062 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10063 }
10064 else
10065 {
10066 uint32_t u32Result = 0;
10067 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10068 if (IOM_SUCCESS(rcStrict))
10069 {
10070 /* Save result of I/O IN instr. in AL/AX/EAX. */
10071 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10072 }
10073 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10074 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10075 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10076 }
10077 }
10078
10079 if (IOM_SUCCESS(rcStrict))
10080 {
10081 if (!fUpdateRipAlready)
10082 {
10083 pMixedCtx->rip += cbInstr;
10084 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10085 }
10086
10087 /* INS & OUTS with REP prefix modify RFLAGS. */
10088 if (fIOString)
10089 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10090
10091 /*
10092 * If any I/O breakpoints are armed, we need to check if one triggered
10093 * and take appropriate action.
10094 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10095 */
10096 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10097 AssertRCReturn(rc2, rc2);
10098
10099 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10100 * execution engines about whether hyper BPs and such are pending. */
10101 uint32_t const uDr7 = pMixedCtx->dr[7];
10102 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10103 && X86_DR7_ANY_RW_IO(uDr7)
10104 && (pMixedCtx->cr4 & X86_CR4_DE))
10105 || DBGFBpIsHwIoArmed(pVM)))
10106 {
10107 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10108
10109 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10110 VMMRZCallRing3Disable(pVCpu);
10111 HM_DISABLE_PREEMPT_IF_NEEDED();
10112
10113 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10114
10115 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10116 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10117 {
10118 /* Raise #DB. */
10119 if (fIsGuestDbgActive)
10120 ASMSetDR6(pMixedCtx->dr[6]);
10121 if (pMixedCtx->dr[7] != uDr7)
10122 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10123
10124 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10125 }
10126 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10127 else if ( rcStrict2 != VINF_SUCCESS
10128 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10129 rcStrict = rcStrict2;
10130
10131 HM_RESTORE_PREEMPT_IF_NEEDED();
10132 VMMRZCallRing3Enable(pVCpu);
10133 }
10134 }
10135
10136#ifdef DEBUG
10137 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10138 Assert(!fIOWrite);
10139 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10140 Assert(fIOWrite);
10141 else
10142 {
10143 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10144 * statuses, that the VMM device and some others may return. See
10145 * IOM_SUCCESS() for guidance. */
10146 AssertMsg( RT_FAILURE(rcStrict)
10147 || rcStrict == VINF_SUCCESS
10148 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10149 || rcStrict == VINF_EM_DBG_BREAKPOINT
10150 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10151 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10152 }
10153#endif
10154
10155 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10156 return VBOXSTRICTRC_TODO(rcStrict);
10157}
10158
10159
10160/**
10161 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10162 * VM-exit.
10163 */
10164HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10165{
10166 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10167
10168 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10169 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10170 AssertRCReturn(rc, rc);
10171 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10172 {
10173 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10174 AssertRCReturn(rc, rc);
10175 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10176 {
10177 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10178
10179 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10180 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10181 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10182 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10183 {
10184 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10185 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10186
10187 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10188 Assert(!pVCpu->hm.s.Event.fPending);
10189 pVCpu->hm.s.Event.fPending = true;
10190 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10191 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10192 AssertRCReturn(rc, rc);
10193 if (fErrorCodeValid)
10194 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10195 else
10196 pVCpu->hm.s.Event.u32ErrCode = 0;
10197 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10198 && uVector == X86_XCPT_PF)
10199 {
10200 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10201 }
10202
10203 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10204 }
10205 }
10206 }
10207
10208 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10209 * emulation. */
10210 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10211 return VERR_EM_INTERPRETER;
10212}
10213
10214
10215/**
10216 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10217 */
10218HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10219{
10220 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10221 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10222 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10223 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10224 AssertRCReturn(rc, rc);
10225 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10226 return VINF_EM_DBG_STEPPED;
10227}
10228
10229
10230/**
10231 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10232 */
10233HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10234{
10235 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10236
10237 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10238 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10239 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10240 return VINF_SUCCESS;
10241 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10242 return rc;
10243
10244#if 0
10245 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10246 * just sync the whole thing. */
10247 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10248#else
10249 /* Aggressive state sync. for now. */
10250 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10251 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10252 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10253#endif
10254 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10255 AssertRCReturn(rc, rc);
10256
10257 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10258 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10259 switch (uAccessType)
10260 {
10261 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10262 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10263 {
10264 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10265 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10266 {
10267 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10268 }
10269
10270 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10271 GCPhys &= PAGE_BASE_GC_MASK;
10272 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10273 PVM pVM = pVCpu->CTX_SUFF(pVM);
10274 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10275 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10276
10277 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10278 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10279 CPUMCTX2CORE(pMixedCtx), GCPhys);
10280 rc = VBOXSTRICTRC_VAL(rc2);
10281 Log4(("ApicAccess rc=%d\n", rc));
10282 if ( rc == VINF_SUCCESS
10283 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10284 || rc == VERR_PAGE_NOT_PRESENT)
10285 {
10286 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10287 | HM_CHANGED_GUEST_RSP
10288 | HM_CHANGED_GUEST_RFLAGS
10289 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10290 rc = VINF_SUCCESS;
10291 }
10292 break;
10293 }
10294
10295 default:
10296 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10297 rc = VINF_EM_RAW_EMULATE_INSTR;
10298 break;
10299 }
10300
10301 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10302 return rc;
10303}
10304
10305
10306/**
10307 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10308 * VM-exit.
10309 */
10310HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10311{
10312 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10313
10314 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10315 if (pVmxTransient->fWasGuestDebugStateActive)
10316 {
10317 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10318 HMVMX_RETURN_UNEXPECTED_EXIT();
10319 }
10320
10321 int rc = VERR_INTERNAL_ERROR_5;
10322 if ( !DBGFIsStepping(pVCpu)
10323 && !pVCpu->hm.s.fSingleInstruction
10324 && !pVmxTransient->fWasHyperDebugStateActive)
10325 {
10326 /* Don't intercept MOV DRx and #DB any more. */
10327 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10328 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10329 AssertRCReturn(rc, rc);
10330
10331 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10332 {
10333#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10334 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10335 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10336 AssertRCReturn(rc, rc);
10337#endif
10338 }
10339
10340 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10341 VMMRZCallRing3Disable(pVCpu);
10342 HM_DISABLE_PREEMPT_IF_NEEDED();
10343
10344 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10345 PVM pVM = pVCpu->CTX_SUFF(pVM);
10346 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10347 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10348
10349 HM_RESTORE_PREEMPT_IF_NEEDED();
10350 VMMRZCallRing3Enable(pVCpu);
10351
10352#ifdef VBOX_WITH_STATISTICS
10353 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10354 AssertRCReturn(rc, rc);
10355 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10357 else
10358 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10359#endif
10360 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10361 return VINF_SUCCESS;
10362 }
10363
10364 /*
10365 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10366 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10367 */
10368 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10369 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10370 AssertRCReturn(rc, rc);
10371 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10372
10373 PVM pVM = pVCpu->CTX_SUFF(pVM);
10374 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10375 {
10376 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10377 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10378 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10379 if (RT_SUCCESS(rc))
10380 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10381 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10382 }
10383 else
10384 {
10385 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10386 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10387 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10389 }
10390
10391 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10392 if (RT_SUCCESS(rc))
10393 {
10394 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10395 AssertRCReturn(rc2, rc2);
10396 }
10397 return rc;
10398}
10399
10400
10401/**
10402 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10403 * Conditional VM-exit.
10404 */
10405HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10406{
10407 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10408 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10409
10410 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10411 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10412 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10413 return VINF_SUCCESS;
10414 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10415 return rc;
10416
10417 RTGCPHYS GCPhys = 0;
10418 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10419
10420#if 0
10421 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10422#else
10423 /* Aggressive state sync. for now. */
10424 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10425 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10426 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10427#endif
10428 AssertRCReturn(rc, rc);
10429
10430 /*
10431 * If we succeed, resume guest execution.
10432 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10433 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10434 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10435 * weird case. See @bugref{6043}.
10436 */
10437 PVM pVM = pVCpu->CTX_SUFF(pVM);
10438 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10439 rc = VBOXSTRICTRC_VAL(rc2);
10440 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10441 if ( rc == VINF_SUCCESS
10442 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10443 || rc == VERR_PAGE_NOT_PRESENT)
10444 {
10445 /* Successfully handled MMIO operation. */
10446 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10447 | HM_CHANGED_GUEST_RSP
10448 | HM_CHANGED_GUEST_RFLAGS
10449 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10450 rc = VINF_SUCCESS;
10451 }
10452 return rc;
10453}
10454
10455
10456/**
10457 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10458 * VM-exit.
10459 */
10460HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10461{
10462 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10463 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10464
10465 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10466 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10467 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10468 return VINF_SUCCESS;
10469 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10470 return rc;
10471
10472 RTGCPHYS GCPhys = 0;
10473 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10474 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10475#if 0
10476 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10477#else
10478 /* Aggressive state sync. for now. */
10479 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10480 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10481 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10482#endif
10483 AssertRCReturn(rc, rc);
10484
10485 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10486 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10487
10488 RTGCUINT uErrorCode = 0;
10489 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10490 uErrorCode |= X86_TRAP_PF_ID;
10491 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10492 uErrorCode |= X86_TRAP_PF_RW;
10493 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10494 uErrorCode |= X86_TRAP_PF_P;
10495
10496 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10497
10498 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10499 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10500
10501 /* Handle the pagefault trap for the nested shadow table. */
10502 PVM pVM = pVCpu->CTX_SUFF(pVM);
10503 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10504 TRPMResetTrap(pVCpu);
10505
10506 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10507 if ( rc == VINF_SUCCESS
10508 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10509 || rc == VERR_PAGE_NOT_PRESENT)
10510 {
10511 /* Successfully synced our nested page tables. */
10512 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10513 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10514 | HM_CHANGED_GUEST_RSP
10515 | HM_CHANGED_GUEST_RFLAGS);
10516 return VINF_SUCCESS;
10517 }
10518
10519 Log4(("EPT return to ring-3 rc=%d\n"));
10520 return rc;
10521}
10522
10523/** @} */
10524
10525/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10526/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10527/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10528
10529/** @name VM-exit exception handlers.
10530 * @{
10531 */
10532
10533/**
10534 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10535 */
10536static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10537{
10538 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10539 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10540
10541 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10542 AssertRCReturn(rc, rc);
10543
10544 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10545 {
10546 /* Old-style FPU error reporting needs some extra work. */
10547 /** @todo don't fall back to the recompiler, but do it manually. */
10548 return VERR_EM_INTERPRETER;
10549 }
10550
10551 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10552 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10553 return rc;
10554}
10555
10556
10557/**
10558 * VM-exit exception handler for #BP (Breakpoint exception).
10559 */
10560static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10561{
10562 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10563 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10564
10565 /** @todo Try optimize this by not saving the entire guest state unless
10566 * really needed. */
10567 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10568 AssertRCReturn(rc, rc);
10569
10570 PVM pVM = pVCpu->CTX_SUFF(pVM);
10571 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10572 if (rc == VINF_EM_RAW_GUEST_TRAP)
10573 {
10574 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10575 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10576 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10577 AssertRCReturn(rc, rc);
10578
10579 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10580 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10581 }
10582
10583 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10584 return rc;
10585}
10586
10587
10588/**
10589 * VM-exit exception handler for #DB (Debug exception).
10590 */
10591static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10592{
10593 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10594 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10595 Log6(("XcptDB\n"));
10596
10597 /*
10598 * Get the DR6-like values from the exit qualification and pass it to DBGF
10599 * for processing.
10600 */
10601 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10602 AssertRCReturn(rc, rc);
10603
10604 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10605 uint64_t uDR6 = X86_DR6_INIT_VAL;
10606 uDR6 |= ( pVmxTransient->uExitQualification
10607 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10608
10609 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10610 if (rc == VINF_EM_RAW_GUEST_TRAP)
10611 {
10612 /*
10613 * The exception was for the guest. Update DR6, DR7.GD and
10614 * IA32_DEBUGCTL.LBR before forwarding it.
10615 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10616 */
10617 VMMRZCallRing3Disable(pVCpu);
10618 HM_DISABLE_PREEMPT_IF_NEEDED();
10619
10620 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10621 pMixedCtx->dr[6] |= uDR6;
10622 if (CPUMIsGuestDebugStateActive(pVCpu))
10623 ASMSetDR6(pMixedCtx->dr[6]);
10624
10625 HM_RESTORE_PREEMPT_IF_NEEDED();
10626 VMMRZCallRing3Enable(pVCpu);
10627
10628 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10629 AssertRCReturn(rc, rc);
10630
10631 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10632 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10633
10634 /* Paranoia. */
10635 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10636 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10637
10638 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10639 AssertRCReturn(rc, rc);
10640
10641 /*
10642 * Raise #DB in the guest.
10643 */
10644 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10645 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10646 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10647 AssertRCReturn(rc, rc);
10648 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10649 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10650 return VINF_SUCCESS;
10651 }
10652
10653 /*
10654 * Not a guest trap, must be a hypervisor related debug event then.
10655 * Update DR6 in case someone is interested in it.
10656 */
10657 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10658 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10659 CPUMSetHyperDR6(pVCpu, uDR6);
10660
10661 return rc;
10662}
10663
10664
10665/**
10666 * VM-exit exception handler for #NM (Device-not-available exception: floating
10667 * point exception).
10668 */
10669static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10670{
10671 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10672
10673 /* We require CR0 and EFER. EFER is always up-to-date. */
10674 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10675 AssertRCReturn(rc, rc);
10676
10677 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10678 VMMRZCallRing3Disable(pVCpu);
10679 HM_DISABLE_PREEMPT_IF_NEEDED();
10680
10681 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10682 if (pVmxTransient->fWasGuestFPUStateActive)
10683 {
10684 rc = VINF_EM_RAW_GUEST_TRAP;
10685 Assert(CPUMIsGuestFPUStateActive(pVCpu) || VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10686 }
10687 else
10688 {
10689#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10690 Assert(!pVmxTransient->fWasGuestFPUStateActive);
10691#endif
10692 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10693 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10694 }
10695
10696 HM_RESTORE_PREEMPT_IF_NEEDED();
10697 VMMRZCallRing3Enable(pVCpu);
10698
10699 if (rc == VINF_SUCCESS)
10700 {
10701 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10702 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10703 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10704 }
10705 else
10706 {
10707 /* Forward #NM to the guest. */
10708 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10709 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10710 AssertRCReturn(rc, rc);
10711 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10712 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10713 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10714 }
10715
10716 return VINF_SUCCESS;
10717}
10718
10719
10720/**
10721 * VM-exit exception handler for #GP (General-protection exception).
10722 *
10723 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10724 */
10725static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10726{
10727 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10728 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10729
10730 int rc = VERR_INTERNAL_ERROR_5;
10731 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10732 {
10733#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10734 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10735 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10736 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10737 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10738 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10739 AssertRCReturn(rc, rc);
10740 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10741 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10742 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10743 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10744 return rc;
10745#else
10746 /* We don't intercept #GP. */
10747 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10748 return VERR_VMX_UNEXPECTED_EXCEPTION;
10749#endif
10750 }
10751
10752 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10753 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10754
10755 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10756 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10757 AssertRCReturn(rc, rc);
10758
10759 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10760 uint32_t cbOp = 0;
10761 PVM pVM = pVCpu->CTX_SUFF(pVM);
10762 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10763 if (RT_SUCCESS(rc))
10764 {
10765 rc = VINF_SUCCESS;
10766 Assert(cbOp == pDis->cbInstr);
10767 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10768 switch (pDis->pCurInstr->uOpcode)
10769 {
10770 case OP_CLI:
10771 {
10772 pMixedCtx->eflags.Bits.u1IF = 0;
10773 pMixedCtx->rip += pDis->cbInstr;
10774 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10775 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10776 break;
10777 }
10778
10779 case OP_STI:
10780 {
10781 pMixedCtx->eflags.Bits.u1IF = 1;
10782 pMixedCtx->rip += pDis->cbInstr;
10783 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10784 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10785 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10786 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10787 break;
10788 }
10789
10790 case OP_HLT:
10791 {
10792 rc = VINF_EM_HALT;
10793 pMixedCtx->rip += pDis->cbInstr;
10794 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10795 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10796 break;
10797 }
10798
10799 case OP_POPF:
10800 {
10801 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10802 uint32_t cbParm = 0;
10803 uint32_t uMask = 0;
10804 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10805 {
10806 cbParm = 4;
10807 uMask = 0xffffffff;
10808 }
10809 else
10810 {
10811 cbParm = 2;
10812 uMask = 0xffff;
10813 }
10814
10815 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10816 RTGCPTR GCPtrStack = 0;
10817 X86EFLAGS Eflags;
10818 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10819 &GCPtrStack);
10820 if (RT_SUCCESS(rc))
10821 {
10822 Assert(sizeof(Eflags.u32) >= cbParm);
10823 Eflags.u32 = 0;
10824 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10825 }
10826 if (RT_FAILURE(rc))
10827 {
10828 rc = VERR_EM_INTERPRETER;
10829 break;
10830 }
10831 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10832 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10833 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10834 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10835 pMixedCtx->esp += cbParm;
10836 pMixedCtx->esp &= uMask;
10837 pMixedCtx->rip += pDis->cbInstr;
10838
10839 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10840 | HM_CHANGED_GUEST_RSP
10841 | HM_CHANGED_GUEST_RFLAGS);
10842 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10843 break;
10844 }
10845
10846 case OP_PUSHF:
10847 {
10848 uint32_t cbParm = 0;
10849 uint32_t uMask = 0;
10850 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10851 {
10852 cbParm = 4;
10853 uMask = 0xffffffff;
10854 }
10855 else
10856 {
10857 cbParm = 2;
10858 uMask = 0xffff;
10859 }
10860
10861 /* Get the stack pointer & push the contents of eflags onto the stack. */
10862 RTGCPTR GCPtrStack = 0;
10863 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10864 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10865 if (RT_FAILURE(rc))
10866 {
10867 rc = VERR_EM_INTERPRETER;
10868 break;
10869 }
10870 X86EFLAGS Eflags = pMixedCtx->eflags;
10871 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10872 Eflags.Bits.u1RF = 0;
10873 Eflags.Bits.u1VM = 0;
10874
10875 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10876 if (RT_FAILURE(rc))
10877 {
10878 rc = VERR_EM_INTERPRETER;
10879 break;
10880 }
10881 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10882 pMixedCtx->esp -= cbParm;
10883 pMixedCtx->esp &= uMask;
10884 pMixedCtx->rip += pDis->cbInstr;
10885 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
10886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10887 break;
10888 }
10889
10890 case OP_IRET:
10891 {
10892 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10893 * instruction reference. */
10894 RTGCPTR GCPtrStack = 0;
10895 uint32_t uMask = 0xffff;
10896 uint16_t aIretFrame[3];
10897 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10898 {
10899 rc = VERR_EM_INTERPRETER;
10900 break;
10901 }
10902 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10903 &GCPtrStack);
10904 if (RT_SUCCESS(rc))
10905 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10906 if (RT_FAILURE(rc))
10907 {
10908 rc = VERR_EM_INTERPRETER;
10909 break;
10910 }
10911 pMixedCtx->eip = 0;
10912 pMixedCtx->ip = aIretFrame[0];
10913 pMixedCtx->cs.Sel = aIretFrame[1];
10914 pMixedCtx->cs.ValidSel = aIretFrame[1];
10915 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10916 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10917 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10918 pMixedCtx->sp += sizeof(aIretFrame);
10919 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10920 | HM_CHANGED_GUEST_SEGMENT_REGS
10921 | HM_CHANGED_GUEST_RSP
10922 | HM_CHANGED_GUEST_RFLAGS);
10923 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10924 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10925 break;
10926 }
10927
10928 case OP_INT:
10929 {
10930 uint16_t uVector = pDis->Param1.uValue & 0xff;
10931 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10932 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10933 break;
10934 }
10935
10936 case OP_INTO:
10937 {
10938 if (pMixedCtx->eflags.Bits.u1OF)
10939 {
10940 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10941 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10942 }
10943 break;
10944 }
10945
10946 default:
10947 {
10948 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10949 EMCODETYPE_SUPERVISOR);
10950 rc = VBOXSTRICTRC_VAL(rc2);
10951 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10952 Log4(("#GP rc=%Rrc\n", rc));
10953 break;
10954 }
10955 }
10956 }
10957 else
10958 rc = VERR_EM_INTERPRETER;
10959
10960 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10961 ("#GP Unexpected rc=%Rrc\n", rc));
10962 return rc;
10963}
10964
10965
10966/**
10967 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10968 * the exception reported in the VMX transient structure back into the VM.
10969 *
10970 * @remarks Requires uExitIntInfo in the VMX transient structure to be
10971 * up-to-date.
10972 */
10973static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10974{
10975 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10976
10977 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10978 hmR0VmxCheckExitDueToEventDelivery(). */
10979 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10980 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10981 AssertRCReturn(rc, rc);
10982 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10983
10984 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10985 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10986 return VINF_SUCCESS;
10987}
10988
10989
10990/**
10991 * VM-exit exception handler for #PF (Page-fault exception).
10992 */
10993static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10994{
10995 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10996 PVM pVM = pVCpu->CTX_SUFF(pVM);
10997 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10998 rc |= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10999 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11000 AssertRCReturn(rc, rc);
11001
11002#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11003 if (pVM->hm.s.fNestedPaging)
11004 {
11005 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11006 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11007 {
11008 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11009 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11010 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11011 }
11012 else
11013 {
11014 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11015 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11016 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11017 }
11018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11019 return rc;
11020 }
11021#else
11022 Assert(!pVM->hm.s.fNestedPaging);
11023#endif
11024
11025 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11026 AssertRCReturn(rc, rc);
11027
11028 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11029 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11030
11031 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11032 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11033 (RTGCPTR)pVmxTransient->uExitQualification);
11034
11035 Log4(("#PF: rc=%Rrc\n", rc));
11036 if (rc == VINF_SUCCESS)
11037 {
11038 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11039 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11040 * memory? We don't update the whole state here... */
11041 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11042 | HM_CHANGED_GUEST_RSP
11043 | HM_CHANGED_GUEST_RFLAGS
11044 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11045 TRPMResetTrap(pVCpu);
11046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11047 return rc;
11048 }
11049 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11050 {
11051 if (!pVmxTransient->fVectoringPF)
11052 {
11053 /* It's a guest page fault and needs to be reflected to the guest. */
11054 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11055 TRPMResetTrap(pVCpu);
11056 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11057 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11058 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11059 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11060 }
11061 else
11062 {
11063 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11064 TRPMResetTrap(pVCpu);
11065 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11066 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11067 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11068 }
11069
11070 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11071 return VINF_SUCCESS;
11072 }
11073
11074 TRPMResetTrap(pVCpu);
11075 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11076 return rc;
11077}
11078
11079/** @} */
11080
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette