VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49274

Last change on this file since 49274 was 49274, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Incorrect assertion for SMP.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 461.0 KB
Line 
1/* $Id: HMVMXR0.cpp 49274 2013-10-24 12:34:28Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#define HMVMX_ALWAYS_FLUSH_TLB
46#endif
47
48
49/*******************************************************************************
50* Defined Constants And Macros *
51*******************************************************************************/
52#if defined(RT_ARCH_AMD64)
53# define HMVMX_IS_64BIT_HOST_MODE() (true)
54typedef RTHCUINTREG HMVMXHCUINTREG;
55#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56extern "C" uint32_t g_fVMXIs64bitHost;
57# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58typedef uint64_t HMVMXHCUINTREG;
59#else
60# define HMVMX_IS_64BIT_HOST_MODE() (false)
61typedef RTHCUINTREG HMVMXHCUINTREG;
62#endif
63
64/** Use the function table. */
65#define HMVMX_USE_FUNCTION_TABLE
66
67/**
68 * The maximum number of MSRs we are willing to swap during a world-switch.
69 * Intel claims 512/check capability MSR, we don't want to do anywhere close
70 * to that. See Intel spec. 24.7.2 "VM-Exit Controls for MSRs"
71 *
72 * Bump this count as and when required, there's no backward compatibility
73 * requirement.
74 */
75#define HMVMX_MAX_SWAP_MSR_COUNT 5
76
77/** Determine which tagged-TLB flush handler to use. */
78#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
79#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
80#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
81#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
82
83/** @name Updated-guest-state flags.
84 * @{ */
85#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
86#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
87#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
88#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
89#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
90#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
91#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
92#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
93#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
94#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
95#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
96#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
97#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
98#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
99#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
100#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
101#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
102#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
103#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
104#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
105#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
106 | HMVMX_UPDATED_GUEST_RSP \
107 | HMVMX_UPDATED_GUEST_RFLAGS \
108 | HMVMX_UPDATED_GUEST_CR0 \
109 | HMVMX_UPDATED_GUEST_CR3 \
110 | HMVMX_UPDATED_GUEST_CR4 \
111 | HMVMX_UPDATED_GUEST_GDTR \
112 | HMVMX_UPDATED_GUEST_IDTR \
113 | HMVMX_UPDATED_GUEST_LDTR \
114 | HMVMX_UPDATED_GUEST_TR \
115 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
116 | HMVMX_UPDATED_GUEST_DEBUG \
117 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
118 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
119 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
120 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
121 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
122 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
123 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
124 | HMVMX_UPDATED_GUEST_APIC_STATE)
125/** @} */
126
127/** @name
128 * Flags to skip redundant reads of some common VMCS fields that are not part of
129 * the guest-CPU state but are in the transient structure.
130 */
131#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
132#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
133#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
134#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
135#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
136#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
137#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
138/** @} */
139
140/** @name
141 * States of the VMCS.
142 *
143 * This does not reflect all possible VMCS states but currently only those
144 * needed for maintaining the VMCS consistently even when thread-context hooks
145 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
146 */
147#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
148#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
149#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
150/** @} */
151
152/**
153 * Exception bitmap mask for real-mode guests (real-on-v86).
154 *
155 * We need to intercept all exceptions manually (except #PF). #NM is also
156 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
157 * even in real-mode if we have Nested Paging support.
158 */
159#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
160 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
161 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
162 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
163 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
164 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
165 | RT_BIT(X86_XCPT_XF))
166
167/**
168 * Exception bitmap mask for all contributory exceptions.
169 *
170 * Page fault is deliberately excluded here as it's conditional as to whether
171 * it's contributory or benign. Page faults are handled separately.
172 */
173#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
174 | RT_BIT(X86_XCPT_DE))
175
176/** Maximum VM-instruction error number. */
177#define HMVMX_INSTR_ERROR_MAX 28
178
179/** Profiling macro. */
180#ifdef HM_PROFILE_EXIT_DISPATCH
181# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
183#else
184# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
185# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
186#endif
187
188/** Assert that preemption is disabled or covered by thread-context hooks. */
189#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
191
192/** Assert that we haven't migrated CPUs when thread-context hooks are not
193 * used. */
194#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
195 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
196 ("Illegal migration! Entered on CPU %u Current %u\n", \
197 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
198
199/** Helper macro for VM-exit handlers called unexpectedly. */
200#define HMVMX_RETURN_UNEXPECTED_EXIT() \
201 do { \
202 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
203 return VERR_VMX_UNEXPECTED_EXIT; \
204 } while (0)
205
206
207/*******************************************************************************
208* Structures and Typedefs *
209*******************************************************************************/
210/**
211 * VMX transient state.
212 *
213 * A state structure for holding miscellaneous information across
214 * VMX non-root operation and restored after the transition.
215 */
216typedef struct VMXTRANSIENT
217{
218 /** The host's rflags/eflags. */
219 RTCCUINTREG uEflags;
220#if HC_ARCH_BITS == 32
221 uint32_t u32Alignment0;
222#endif
223 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
224 uint64_t u64LStarMsr;
225 /** The guest's TPR value used for TPR shadowing. */
226 uint8_t u8GuestTpr;
227 /** Alignment. */
228 uint8_t abAlignment0[7];
229
230 /** The basic VM-exit reason. */
231 uint16_t uExitReason;
232 /** Alignment. */
233 uint16_t u16Alignment0;
234 /** The VM-exit interruption error code. */
235 uint32_t uExitIntErrorCode;
236 /** The VM-exit exit qualification. */
237 uint64_t uExitQualification;
238
239 /** The VM-exit interruption-information field. */
240 uint32_t uExitIntInfo;
241 /** The VM-exit instruction-length field. */
242 uint32_t cbInstr;
243 /** The VM-exit instruction-information field. */
244 union
245 {
246 /** Plain unsigned int representation. */
247 uint32_t u;
248 /** INS and OUTS information. */
249 struct
250 {
251 uint32_t u6Reserved0 : 7;
252 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253 uint32_t u3AddrSize : 3;
254 uint32_t u5Reserved1 : 5;
255 /** The segment register (X86_SREG_XXX). */
256 uint32_t iSegReg : 3;
257 uint32_t uReserved2 : 14;
258 } StrIo;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest FPU was active at the time of VM-exit. */
281 bool fWasGuestFPUStateActive;
282 /** Whether the guest debug state was active at the time of VM-exit. */
283 bool fWasGuestDebugStateActive;
284 /** Whether the hyper debug state was active at the time of VM-exit. */
285 bool fWasHyperDebugStateActive;
286 /** Whether TSC-offsetting should be setup before VM-entry. */
287 bool fUpdateTscOffsettingAndPreemptTimer;
288 /** Whether the VM-exit was caused by a page-fault during delivery of a
289 * contributory exception or a page-fault. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322
323/**
324 * VMX VM-exit handler.
325 *
326 * @returns VBox status code.
327 * @param pVCpu Pointer to the VMCPU.
328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
329 * out-of-sync. Make sure to update the required
330 * fields before using them.
331 * @param pVmxTransient Pointer to the VMX-transient structure.
332 */
333#ifndef HMVMX_USE_FUNCTION_TABLE
334typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335#else
336typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
337/** Pointer to VM-exit handler. */
338typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
339#endif
340
341
342/*******************************************************************************
343* Internal Functions *
344*******************************************************************************/
345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
346static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
347static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
348static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
350#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
351static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
352#endif
353#ifndef HMVMX_USE_FUNCTION_TABLE
354DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
355# define HMVMX_EXIT_DECL static int
356#else
357# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
358#endif
359
360/** @name VM-exit handlers.
361 * @{
362 */
363static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
364static FNVMXEXITHANDLER hmR0VmxExitExtInt;
365static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
366static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
367static FNVMXEXITHANDLER hmR0VmxExitSipi;
368static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
369static FNVMXEXITHANDLER hmR0VmxExitSmi;
370static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
371static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
372static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
373static FNVMXEXITHANDLER hmR0VmxExitCpuid;
374static FNVMXEXITHANDLER hmR0VmxExitGetsec;
375static FNVMXEXITHANDLER hmR0VmxExitHlt;
376static FNVMXEXITHANDLER hmR0VmxExitInvd;
377static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
378static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
379static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
380static FNVMXEXITHANDLER hmR0VmxExitRsm;
381static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
382static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
383static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
384static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
385static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
386static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
387static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
388static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
389static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
390static FNVMXEXITHANDLER hmR0VmxExitMwait;
391static FNVMXEXITHANDLER hmR0VmxExitMtf;
392static FNVMXEXITHANDLER hmR0VmxExitMonitor;
393static FNVMXEXITHANDLER hmR0VmxExitPause;
394static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
395static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
396static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
397static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
398static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
404static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407/** @} */
408
409static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
417
418/*******************************************************************************
419* Global Variables *
420*******************************************************************************/
421#ifdef HMVMX_USE_FUNCTION_TABLE
422
423/**
424 * VMX_EXIT dispatch table.
425 */
426static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
427{
428 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
429 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
430 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
431 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
432 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
433 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
434 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
435 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
436 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
437 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
438 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
439 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
440 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
441 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
442 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
443 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
444 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
445 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
446 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
447 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
448 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
449 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
450 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
451 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
452 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
453 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
454 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
455 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
456 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
457 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
458 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
459 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
460 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
461 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
462 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
463 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
464 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
465 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
466 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
468 /* 40 UNDEFINED */ hmR0VmxExitPause,
469 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
470 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
471 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
472 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
473 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
474 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
475 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
476 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
477 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
478 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
479 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
480 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
481 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
482 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
483 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
484 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
485 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
486 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
487 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
488};
489#endif /* HMVMX_USE_FUNCTION_TABLE */
490
491#ifdef VBOX_STRICT
492static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
493{
494 /* 0 */ "(Not Used)",
495 /* 1 */ "VMCALL executed in VMX root operation.",
496 /* 2 */ "VMCLEAR with invalid physical address.",
497 /* 3 */ "VMCLEAR with VMXON pointer.",
498 /* 4 */ "VMLAUNCH with non-clear VMCS.",
499 /* 5 */ "VMRESUME with non-launched VMCS.",
500 /* 6 */ "VMRESUME after VMXOFF",
501 /* 7 */ "VM entry with invalid control fields.",
502 /* 8 */ "VM entry with invalid host state fields.",
503 /* 9 */ "VMPTRLD with invalid physical address.",
504 /* 10 */ "VMPTRLD with VMXON pointer.",
505 /* 11 */ "VMPTRLD with incorrect revision identifier.",
506 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
507 /* 13 */ "VMWRITE to read-only VMCS component.",
508 /* 14 */ "(Not Used)",
509 /* 15 */ "VMXON executed in VMX root operation.",
510 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
511 /* 17 */ "VM entry with non-launched executing VMCS.",
512 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
513 /* 19 */ "VMCALL with non-clear VMCS.",
514 /* 20 */ "VMCALL with invalid VM-exit control fields.",
515 /* 21 */ "(Not Used)",
516 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
517 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
518 /* 24 */ "VMCALL with invalid SMM-monitor features.",
519 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
520 /* 26 */ "VM entry with events blocked by MOV SS.",
521 /* 27 */ "(Not Used)",
522 /* 28 */ "Invalid operand to INVEPT/INVVPID."
523};
524#endif /* VBOX_STRICT */
525
526
527
528/**
529 * Updates the VM's last error record. If there was a VMX instruction error,
530 * reads the error data from the VMCS and updates VCPU's last error record as
531 * well.
532 *
533 * @param pVM Pointer to the VM.
534 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
535 * VERR_VMX_UNABLE_TO_START_VM or
536 * VERR_VMX_INVALID_VMCS_FIELD).
537 * @param rc The error code.
538 */
539static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
540{
541 AssertPtr(pVM);
542 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
543 || rc == VERR_VMX_UNABLE_TO_START_VM)
544 {
545 AssertPtrReturnVoid(pVCpu);
546 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
547 }
548 pVM->hm.s.lLastError = rc;
549}
550
551
552/**
553 * Reads the VM-entry interruption-information field from the VMCS into the VMX
554 * transient structure.
555 *
556 * @returns VBox status code.
557 * @param pVmxTransient Pointer to the VMX transient structure.
558 *
559 * @remarks No-long-jump zone!!!
560 */
561DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
562{
563 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
564 AssertRCReturn(rc, rc);
565 return VINF_SUCCESS;
566}
567
568
569/**
570 * Reads the VM-entry exception error code field from the VMCS into
571 * the VMX transient structure.
572 *
573 * @returns VBox status code.
574 * @param pVmxTransient Pointer to the VMX transient structure.
575 *
576 * @remarks No-long-jump zone!!!
577 */
578DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
579{
580 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
581 AssertRCReturn(rc, rc);
582 return VINF_SUCCESS;
583}
584
585
586/**
587 * Reads the VM-entry exception error code field from the VMCS into
588 * the VMX transient structure.
589 *
590 * @returns VBox status code.
591 * @param pVCpu Pointer to the VMCPU.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-exit interruption-information field from the VMCS into the VMX
606 * transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVCpu Pointer to the VMCPU.
610 * @param pVmxTransient Pointer to the VMX transient structure.
611 */
612DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
613{
614 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
615 {
616 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
617 AssertRCReturn(rc, rc);
618 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
619 }
620 return VINF_SUCCESS;
621}
622
623
624/**
625 * Reads the VM-exit interruption error code from the VMCS into the VMX
626 * transient structure.
627 *
628 * @returns VBox status code.
629 * @param pVCpu Pointer to the VMCPU.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 */
632DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
633{
634 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
635 {
636 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
637 AssertRCReturn(rc, rc);
638 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
639 }
640 return VINF_SUCCESS;
641}
642
643
644/**
645 * Reads the VM-exit instruction length field from the VMCS into the VMX
646 * transient structure.
647 *
648 * @returns VBox status code.
649 * @param pVCpu Pointer to the VMCPU.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
657 AssertRCReturn(rc, rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit instruction-information field from the VMCS into
666 * the VMX transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVCpu The cross context per CPU structure.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit qualification from the VMCS into the VMX transient structure.
686 *
687 * @returns VBox status code.
688 * @param pVCpu Pointer to the VMCPU.
689 * @param pVmxTransient Pointer to the VMX transient structure.
690 */
691DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
692{
693 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
694 {
695 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
696 AssertRCReturn(rc, rc);
697 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
698 }
699 return VINF_SUCCESS;
700}
701
702
703/**
704 * Reads the IDT-vectoring information field from the VMCS into the VMX
705 * transient structure.
706 *
707 * @returns VBox status code.
708 * @param pVmxTransient Pointer to the VMX transient structure.
709 *
710 * @remarks No-long-jump zone!!!
711 */
712DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
713{
714 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
715 {
716 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
717 AssertRCReturn(rc, rc);
718 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
719 }
720 return VINF_SUCCESS;
721}
722
723
724/**
725 * Reads the IDT-vectoring error code from the VMCS into the VMX
726 * transient structure.
727 *
728 * @returns VBox status code.
729 * @param pVmxTransient Pointer to the VMX transient structure.
730 */
731DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
732{
733 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
734 {
735 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
736 AssertRCReturn(rc, rc);
737 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
738 }
739 return VINF_SUCCESS;
740}
741
742
743/**
744 * Enters VMX root mode operation on the current CPU.
745 *
746 * @returns VBox status code.
747 * @param pVM Pointer to the VM (optional, can be NULL, after
748 * a resume).
749 * @param HCPhysCpuPage Physical address of the VMXON region.
750 * @param pvCpuPage Pointer to the VMXON region.
751 */
752static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
753{
754 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
755 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 if (pVM)
759 {
760 /* Write the VMCS revision dword to the VMXON region. */
761 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
762 }
763
764 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
765 RTCCUINTREG uEflags = ASMIntDisableFlags();
766
767 /* Enable the VMX bit in CR4 if necessary. */
768 RTCCUINTREG uCr4 = ASMGetCR4();
769 if (!(uCr4 & X86_CR4_VMXE))
770 ASMSetCR4(uCr4 | X86_CR4_VMXE);
771
772 /* Enter VMX root mode. */
773 int rc = VMXEnable(HCPhysCpuPage);
774 if (RT_FAILURE(rc))
775 ASMSetCR4(uCr4);
776
777 /* Restore interrupts. */
778 ASMSetFlags(uEflags);
779 return rc;
780}
781
782
783/**
784 * Exits VMX root mode operation on the current CPU.
785 *
786 * @returns VBox status code.
787 */
788static int hmR0VmxLeaveRootMode(void)
789{
790 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
791
792 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
793 RTCCUINTREG uEflags = ASMIntDisableFlags();
794
795 /* If we're for some reason not in VMX root mode, then don't leave it. */
796 RTCCUINTREG uHostCR4 = ASMGetCR4();
797
798 int rc;
799 if (uHostCR4 & X86_CR4_VMXE)
800 {
801 /* Exit VMX root mode and clear the VMX bit in CR4. */
802 VMXDisable();
803 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
804 rc = VINF_SUCCESS;
805 }
806 else
807 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
808
809 /* Restore interrupts. */
810 ASMSetFlags(uEflags);
811 return rc;
812}
813
814
815/**
816 * Allocates and maps one physically contiguous page. The allocated page is
817 * zero'd out. (Used by various VT-x structures).
818 *
819 * @returns IPRT status code.
820 * @param pMemObj Pointer to the ring-0 memory object.
821 * @param ppVirt Where to store the virtual address of the
822 * allocation.
823 * @param pPhys Where to store the physical address of the
824 * allocation.
825 */
826DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
827{
828 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
829 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
830 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
831
832 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
833 if (RT_FAILURE(rc))
834 return rc;
835 *ppVirt = RTR0MemObjAddress(*pMemObj);
836 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
837 ASMMemZero32(*ppVirt, PAGE_SIZE);
838 return VINF_SUCCESS;
839}
840
841
842/**
843 * Frees and unmaps an allocated physical page.
844 *
845 * @param pMemObj Pointer to the ring-0 memory object.
846 * @param ppVirt Where to re-initialize the virtual address of
847 * allocation as 0.
848 * @param pHCPhys Where to re-initialize the physical address of the
849 * allocation as 0.
850 */
851DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
852{
853 AssertPtr(pMemObj);
854 AssertPtr(ppVirt);
855 AssertPtr(pHCPhys);
856 if (*pMemObj != NIL_RTR0MEMOBJ)
857 {
858 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
859 AssertRC(rc);
860 *pMemObj = NIL_RTR0MEMOBJ;
861 *ppVirt = 0;
862 *pHCPhys = 0;
863 }
864}
865
866
867/**
868 * Worker function to free VT-x related structures.
869 *
870 * @returns IPRT status code.
871 * @param pVM Pointer to the VM.
872 */
873static void hmR0VmxStructsFree(PVM pVM)
874{
875 for (VMCPUID i = 0; i < pVM->cCpus; i++)
876 {
877 PVMCPU pVCpu = &pVM->aCpus[i];
878 AssertPtr(pVCpu);
879
880#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
881 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
883#endif
884
885 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
887
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
890 }
891
892 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
893#ifdef VBOX_WITH_CRASHDUMP_MAGIC
894 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
895#endif
896}
897
898
899/**
900 * Worker function to allocate VT-x related VM structures.
901 *
902 * @returns IPRT status code.
903 * @param pVM Pointer to the VM.
904 */
905static int hmR0VmxStructsAlloc(PVM pVM)
906{
907 /*
908 * Initialize members up-front so we can cleanup properly on allocation failure.
909 */
910#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
911 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
912 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
913 pVM->hm.s.vmx.HCPhys##a_Name = 0;
914
915#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
919
920#ifdef VBOX_WITH_CRASHDUMP_MAGIC
921 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
922#endif
923 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
924
925 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
926 for (VMCPUID i = 0; i < pVM->cCpus; i++)
927 {
928 PVMCPU pVCpu = &pVM->aCpus[i];
929 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
931 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
932#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935#endif
936 }
937#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
938#undef VMXLOCAL_INIT_VM_MEMOBJ
939
940 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
941 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
942 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
943 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
944
945 /*
946 * Allocate all the VT-x structures.
947 */
948 int rc = VINF_SUCCESS;
949#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951 if (RT_FAILURE(rc))
952 goto cleanup;
953 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
954 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
955#endif
956
957 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
958 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
959 {
960 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
961 &pVM->hm.s.vmx.HCPhysApicAccess);
962 if (RT_FAILURE(rc))
963 goto cleanup;
964 }
965
966 /*
967 * Initialize per-VCPU VT-x structures.
968 */
969 for (VMCPUID i = 0; i < pVM->cCpus; i++)
970 {
971 PVMCPU pVCpu = &pVM->aCpus[i];
972 AssertPtr(pVCpu);
973
974 /* Allocate the VM control structure (VMCS). */
975 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978
979 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
980 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
981 {
982 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
983 &pVCpu->hm.s.vmx.HCPhysVirtApic);
984 if (RT_FAILURE(rc))
985 goto cleanup;
986 }
987
988 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
990 {
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
992 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
996 }
997
998#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
999 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003
1004 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1005 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008#endif
1009 }
1010
1011 return VINF_SUCCESS;
1012
1013cleanup:
1014 hmR0VmxStructsFree(pVM);
1015 return rc;
1016}
1017
1018
1019/**
1020 * Does global VT-x initialization (called during module initialization).
1021 *
1022 * @returns VBox status code.
1023 */
1024VMMR0DECL(int) VMXR0GlobalInit(void)
1025{
1026#ifdef HMVMX_USE_FUNCTION_TABLE
1027 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1028# ifdef VBOX_STRICT
1029 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1030 Assert(g_apfnVMExitHandlers[i]);
1031# endif
1032#endif
1033 return VINF_SUCCESS;
1034}
1035
1036
1037/**
1038 * Does global VT-x termination (called during module termination).
1039 */
1040VMMR0DECL(void) VMXR0GlobalTerm()
1041{
1042 /* Nothing to do currently. */
1043}
1044
1045
1046/**
1047 * Sets up and activates VT-x on the current CPU.
1048 *
1049 * @returns VBox status code.
1050 * @param pCpu Pointer to the global CPU info struct.
1051 * @param pVM Pointer to the VM (can be NULL after a host resume
1052 * operation).
1053 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1054 * fEnabledByHost is true).
1055 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1056 * @a fEnabledByHost is true).
1057 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1058 * enable VT-x on the host.
1059 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1060 */
1061VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1062 void *pvMsrs)
1063{
1064 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1065 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067
1068 /* Enable VT-x if it's not already enabled by the host. */
1069 if (!fEnabledByHost)
1070 {
1071 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1072 if (RT_FAILURE(rc))
1073 return rc;
1074 }
1075
1076 /*
1077 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1078 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1079 */
1080 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1081 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1082 {
1083 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1084 pCpu->fFlushAsidBeforeUse = false;
1085 }
1086 else
1087 pCpu->fFlushAsidBeforeUse = true;
1088
1089 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1090 ++pCpu->cTlbFlushes;
1091
1092 return VINF_SUCCESS;
1093}
1094
1095
1096/**
1097 * Deactivates VT-x on the current CPU.
1098 *
1099 * @returns VBox status code.
1100 * @param pCpu Pointer to the global CPU info struct.
1101 * @param pvCpuPage Pointer to the VMXON region.
1102 * @param HCPhysCpuPage Physical address of the VMXON region.
1103 *
1104 * @remarks This function should never be called when SUPR0EnableVTx() or
1105 * similar was used to enable VT-x on the host.
1106 */
1107VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1108{
1109 NOREF(pCpu);
1110 NOREF(pvCpuPage);
1111 NOREF(HCPhysCpuPage);
1112
1113 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1114 return hmR0VmxLeaveRootMode();
1115}
1116
1117
1118/**
1119 * Sets the permission bits for the specified MSR in the MSR bitmap.
1120 *
1121 * @param pVCpu Pointer to the VMCPU.
1122 * @param uMSR The MSR value.
1123 * @param enmRead Whether reading this MSR causes a VM-exit.
1124 * @param enmWrite Whether writing this MSR causes a VM-exit.
1125 */
1126static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1127{
1128 int32_t iBit;
1129 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1130
1131 /*
1132 * Layout:
1133 * 0x000 - 0x3ff - Low MSR read bits
1134 * 0x400 - 0x7ff - High MSR read bits
1135 * 0x800 - 0xbff - Low MSR write bits
1136 * 0xc00 - 0xfff - High MSR write bits
1137 */
1138 if (uMsr <= 0x00001FFF)
1139 iBit = uMsr;
1140 else if ( uMsr >= 0xC0000000
1141 && uMsr <= 0xC0001FFF)
1142 {
1143 iBit = (uMsr - 0xC0000000);
1144 pbMsrBitmap += 0x400;
1145 }
1146 else
1147 {
1148 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1149 return;
1150 }
1151
1152 Assert(iBit <= 0x1fff);
1153 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154 ASMBitSet(pbMsrBitmap, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap, iBit);
1157
1158 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160 else
1161 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162}
1163
1164
1165#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1166/**
1167 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1168 * area.
1169 *
1170 * @returns VBox status code.
1171 * @param pVCpu Pointer to the VMCPU.
1172 * @param cMsrs The number of MSRs.
1173 */
1174DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1175{
1176 /* Update the VCPU's copy of the guest MSR count. */
1177 pVCpu->hm.s.vmx.cGuestMsrs = cMsrs;
1178
1179 /* Update number of guest MSRs to load/store across the world-switch. */
1180 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1181 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1182
1183 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1184 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1185 return VINF_SUCCESS;
1186}
1187
1188
1189/**
1190 * Adds a guest/host MSR pair to be swapped during the world-switch as
1191 * part of the auto-load/store MSR area in the VMCS.
1192 *
1193 * @returns VBox status code.
1194 * @param pVCpu Pointer to the VMCPU.
1195 * @param uMsr The MSR.
1196 * @param uGuestMsr Value of the guest MSR.
1197 * @param uHostMsr Value of the host MSR.
1198 */
1199static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, uint64_t uHostMsrValue)
1200{
1201 AssertMsg(HMVMX_MAX_SWAP_MSR_COUNT < MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc),
1202 ("MSR swap count exceeded. Cpu reports %#RX32, our limit %#RX32\n",
1203 MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc), HMVMX_MAX_SWAP_MSR_COUNT));
1204
1205 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1206 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1207 uint32_t i;
1208 for (i = 0; i < cGuestMsrs; i++)
1209 {
1210 if (pGuestMsr->u32Msr == uMsr)
1211 break;
1212 pGuestMsr++;
1213 }
1214
1215 AssertReturn(i < HMVMX_MAX_SWAP_MSR_COUNT, VERR_HM_MSR_SWAP_COUNT_EXCEEDED);
1216 if (i == cGuestMsrs)
1217 {
1218 ++cGuestMsrs;
1219 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1220 if (RT_UNLIKELY(cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc)))
1221 {
1222 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
1223 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1224 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1225 }
1226
1227 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1228 AssertRCReturn(rc, rc);
1229 }
1230
1231 /* Update the MSR values in the auto-load/store MSR area. */
1232 pGuestMsr->u32Msr = uMsr;
1233 pGuestMsr->u64Value = uGuestMsrValue;
1234
1235 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1236 pHostMsr += i;
1237 pHostMsr->u32Msr = uMsr;
1238 pHostMsr->u64Value = uHostMsrValue;
1239
1240 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1241 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
1242 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1243 return VINF_SUCCESS;
1244}
1245
1246
1247/**
1248 * Removes a guest/shost MSR pair to be swapped during the world-switch from the
1249 * auto-load/store MSR area in the VMCS.
1250 *
1251 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1252 * area.
1253 *
1254 * @returns VBox status code.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 */
1258static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cGuestMsrs; i++)
1264 {
1265 /* Find the MSR. */
1266 if (pGuestMsr->u32Msr == uMsr)
1267 {
1268 /* If it's the last MSR, simply reduce the count. */
1269 if (i == cGuestMsrs - 1)
1270 {
1271 --cGuestMsrs;
1272 break;
1273 }
1274
1275 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1276 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1277 pLastGuestMsr += cGuestMsrs;
1278 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1279 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1280
1281 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1282 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1283 pLastHostMsr += cGuestMsrs;
1284 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1285 pHostMsr->u64Value = pLastHostMsr->u64Value;
1286 --cGuestMsrs;
1287 break;
1288 }
1289 pGuestMsr++;
1290 }
1291
1292 /* Update the VMCS if the count changed (meaning the MSR was found). */
1293 if (cGuestMsrs != pVCpu->hm.s.vmx.cGuestMsrs)
1294 {
1295 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1296 AssertRCReturn(rc, rc);
1297 }
1298
1299 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1300 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1301 return VINF_SUCCESS;
1302}
1303
1304
1305/**
1306 * Updates the value of a host MSR in the auto-load/store area in the VMCS.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxUpdateAutoLoadStoreHostMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1316
1317 for (uint32_t i = 0; i < cMsrs; i++)
1318 {
1319 if (pHostMsr->u32Msr == uMsr)
1320 {
1321 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1322 return VINF_SUCCESS;
1323 }
1324 }
1325
1326 return VERR_NOT_FOUND;
1327}
1328
1329#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1330
1331
1332/**
1333 * Flushes the TLB using EPT.
1334 *
1335 * @returns VBox status code.
1336 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1337 * enmFlush).
1338 * @param enmFlush Type of flush.
1339 *
1340 * @remarks Caller is responsible for making sure this function is called only
1341 * when NestedPaging is supported and providing @a enmFlush that is
1342 * supported by the CPU.
1343 * @remarks Can be called with interrupts disabled.
1344 */
1345static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1346{
1347 uint64_t au64Descriptor[2];
1348 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1349 au64Descriptor[0] = 0;
1350 else
1351 {
1352 Assert(pVCpu);
1353 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1354 }
1355 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1356
1357 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1358 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1359 rc));
1360 if ( RT_SUCCESS(rc)
1361 && pVCpu)
1362 {
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1364 }
1365}
1366
1367
1368/**
1369 * Flushes the TLB using VPID.
1370 *
1371 * @returns VBox status code.
1372 * @param pVM Pointer to the VM.
1373 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1374 * enmFlush).
1375 * @param enmFlush Type of flush.
1376 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1377 * on @a enmFlush).
1378 *
1379 * @remarks Can be called with interrupts disabled.
1380 */
1381static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1382{
1383 AssertPtr(pVM);
1384 Assert(pVM->hm.s.vmx.fVpid);
1385
1386 uint64_t au64Descriptor[2];
1387 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1388 {
1389 au64Descriptor[0] = 0;
1390 au64Descriptor[1] = 0;
1391 }
1392 else
1393 {
1394 AssertPtr(pVCpu);
1395 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1396 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1397 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1398 au64Descriptor[1] = GCPtr;
1399 }
1400
1401 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1402 AssertMsg(rc == VINF_SUCCESS,
1403 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1404 if ( RT_SUCCESS(rc)
1405 && pVCpu)
1406 {
1407 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1408 }
1409}
1410
1411
1412/**
1413 * Invalidates a guest page by guest virtual address. Only relevant for
1414 * EPT/VPID, otherwise there is nothing really to invalidate.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param GCVirt Guest virtual address of the page to invalidate.
1420 */
1421VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1422{
1423 AssertPtr(pVM);
1424 AssertPtr(pVCpu);
1425 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1426
1427 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1428 if (!fFlushPending)
1429 {
1430 /*
1431 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1432 * See @bugref{6043} and @bugref{6177}.
1433 *
1434 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1435 * function maybe called in a loop with individual addresses.
1436 */
1437 if (pVM->hm.s.vmx.fVpid)
1438 {
1439 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1440 {
1441 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1442 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1443 }
1444 else
1445 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1446 }
1447 else if (pVM->hm.s.fNestedPaging)
1448 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1449 }
1450
1451 return VINF_SUCCESS;
1452}
1453
1454
1455/**
1456 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1457 * otherwise there is nothing really to invalidate.
1458 *
1459 * @returns VBox status code.
1460 * @param pVM Pointer to the VM.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param GCPhys Guest physical address of the page to invalidate.
1463 */
1464VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1465{
1466 LogFlowFunc(("%RGp\n", GCPhys));
1467
1468 /*
1469 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1470 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1471 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1472 */
1473 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1474 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1475 return VINF_SUCCESS;
1476}
1477
1478
1479/**
1480 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1481 * case where neither EPT nor VPID is supported by the CPU.
1482 *
1483 * @param pVM Pointer to the VM.
1484 * @param pVCpu Pointer to the VMCPU.
1485 * @param pCpu Pointer to the global HM struct.
1486 *
1487 * @remarks Called with interrupts disabled.
1488 */
1489static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1490{
1491 AssertPtr(pVCpu);
1492 AssertPtr(pCpu);
1493 NOREF(pVM);
1494
1495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1496
1497 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1498#if 0
1499 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1500 pVCpu->hm.s.TlbShootdown.cPages = 0;
1501#endif
1502
1503 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1504 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1505 pVCpu->hm.s.fForceTLBFlush = false;
1506 return;
1507}
1508
1509
1510/**
1511 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1512 *
1513 * @param pVM Pointer to the VM.
1514 * @param pVCpu Pointer to the VMCPU.
1515 * @param pCpu Pointer to the global HM CPU struct.
1516 * @remarks All references to "ASID" in this function pertains to "VPID" in
1517 * Intel's nomenclature. The reason is, to avoid confusion in compare
1518 * statements since the host-CPU copies are named "ASID".
1519 *
1520 * @remarks Called with interrupts disabled.
1521 */
1522static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1523{
1524#ifdef VBOX_WITH_STATISTICS
1525 bool fTlbFlushed = false;
1526# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1527# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1528 if (!fTlbFlushed) \
1529 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1530 } while (0)
1531#else
1532# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1533# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1534#endif
1535
1536 AssertPtr(pVM);
1537 AssertPtr(pCpu);
1538 AssertPtr(pVCpu);
1539 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1540 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1541 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1542
1543
1544 /*
1545 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1546 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1547 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1548 */
1549 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1550 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1551 {
1552 ++pCpu->uCurrentAsid;
1553 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1554 {
1555 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1556 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1557 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1558 }
1559
1560 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1561 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1562 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1563
1564 /*
1565 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1566 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1567 */
1568 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1569 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1570 HMVMX_SET_TAGGED_TLB_FLUSHED();
1571 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1572 }
1573
1574 /* Check for explicit TLB shootdowns. */
1575 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1576 {
1577 /*
1578 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1579 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1580 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1581 * but not guest-physical mappings.
1582 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1583 */
1584 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1585 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1586 HMVMX_SET_TAGGED_TLB_FLUSHED();
1587 }
1588
1589 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1590 * where it is commented out. Support individual entry flushing
1591 * someday. */
1592#if 0
1593 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1594 {
1595 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1596
1597 /*
1598 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1599 * as supported by the CPU.
1600 */
1601 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1602 {
1603 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1604 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1605 }
1606 else
1607 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1608
1609 HMVMX_SET_TAGGED_TLB_FLUSHED();
1610 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1611 pVCpu->hm.s.TlbShootdown.cPages = 0;
1612 }
1613#endif
1614
1615 pVCpu->hm.s.fForceTLBFlush = false;
1616
1617 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1618
1619 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1620 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1621 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1622 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1623 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1624 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1625 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1626 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1627
1628 /* Update VMCS with the VPID. */
1629 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1630 AssertRC(rc);
1631
1632#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1633}
1634
1635
1636/**
1637 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1638 *
1639 * @returns VBox status code.
1640 * @param pVM Pointer to the VM.
1641 * @param pVCpu Pointer to the VMCPU.
1642 * @param pCpu Pointer to the global HM CPU struct.
1643 *
1644 * @remarks Called with interrupts disabled.
1645 */
1646static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1647{
1648 AssertPtr(pVM);
1649 AssertPtr(pVCpu);
1650 AssertPtr(pCpu);
1651 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1652 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1653
1654 /*
1655 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1656 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1657 */
1658 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1659 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1660 {
1661 pVCpu->hm.s.fForceTLBFlush = true;
1662 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1663 }
1664
1665 /* Check for explicit TLB shootdown flushes. */
1666 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1667 {
1668 pVCpu->hm.s.fForceTLBFlush = true;
1669 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1670 }
1671
1672 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1673 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1674
1675 if (pVCpu->hm.s.fForceTLBFlush)
1676 {
1677 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1678 pVCpu->hm.s.fForceTLBFlush = false;
1679 }
1680 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1681 * where it is commented out. Support individual entry flushing
1682 * someday. */
1683#if 0
1684 else
1685 {
1686 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1687 {
1688 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1689 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1690 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1691 }
1692 else
1693 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1694
1695 pVCpu->hm.s.TlbShootdown.cPages = 0;
1696 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1697 }
1698#endif
1699}
1700
1701
1702/**
1703 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1704 *
1705 * @returns VBox status code.
1706 * @param pVM Pointer to the VM.
1707 * @param pVCpu Pointer to the VMCPU.
1708 * @param pCpu Pointer to the global HM CPU struct.
1709 *
1710 * @remarks Called with interrupts disabled.
1711 */
1712static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1713{
1714 AssertPtr(pVM);
1715 AssertPtr(pVCpu);
1716 AssertPtr(pCpu);
1717 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1718 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1719
1720 /*
1721 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1722 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1723 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1724 */
1725 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1726 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1727 {
1728 pVCpu->hm.s.fForceTLBFlush = true;
1729 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1730 }
1731
1732 /* Check for explicit TLB shootdown flushes. */
1733 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1734 {
1735 /*
1736 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1737 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1738 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1739 */
1740 pVCpu->hm.s.fForceTLBFlush = true;
1741 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1742 }
1743
1744 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1745 if (pVCpu->hm.s.fForceTLBFlush)
1746 {
1747 ++pCpu->uCurrentAsid;
1748 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1749 {
1750 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1751 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1752 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1753 }
1754
1755 pVCpu->hm.s.fForceTLBFlush = false;
1756 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1757 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1758 if (pCpu->fFlushAsidBeforeUse)
1759 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1760 }
1761 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1762 * where it is commented out. Support individual entry flushing
1763 * someday. */
1764#if 0
1765 else
1766 {
1767 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1768 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1769 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1770 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1771
1772 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1773 {
1774 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1776 {
1777 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1778 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1779 }
1780 else
1781 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1782
1783 pVCpu->hm.s.TlbShootdown.cPages = 0;
1784 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1785 }
1786 else
1787 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1788 }
1789#endif
1790
1791 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1792 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1793 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1794 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1795 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1796 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1797
1798 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1799 AssertRC(rc);
1800}
1801
1802
1803/**
1804 * Flushes the guest TLB entry based on CPU capabilities.
1805 *
1806 * @param pVCpu Pointer to the VMCPU.
1807 * @param pCpu Pointer to the global HM CPU struct.
1808 */
1809DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1810{
1811#ifdef HMVMX_ALWAYS_FLUSH_TLB
1812 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1813#endif
1814 PVM pVM = pVCpu->CTX_SUFF(pVM);
1815 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1816 {
1817 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1818 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1819 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1820 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1821 default:
1822 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1823 break;
1824 }
1825
1826 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
1827 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
1828
1829 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
1830}
1831
1832
1833/**
1834 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1835 * TLB entries from the host TLB before VM-entry.
1836 *
1837 * @returns VBox status code.
1838 * @param pVM Pointer to the VM.
1839 */
1840static int hmR0VmxSetupTaggedTlb(PVM pVM)
1841{
1842 /*
1843 * Determine optimal flush type for Nested Paging.
1844 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1845 * guest execution (see hmR3InitFinalizeR0()).
1846 */
1847 if (pVM->hm.s.fNestedPaging)
1848 {
1849 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1850 {
1851 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1852 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1853 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1854 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1855 else
1856 {
1857 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1858 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1859 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1860 }
1861
1862 /* Make sure the write-back cacheable memory type for EPT is supported. */
1863 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1864 {
1865 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1866 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1867 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1868 }
1869 }
1870 else
1871 {
1872 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1873 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1874 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1875 }
1876 }
1877
1878 /*
1879 * Determine optimal flush type for VPID.
1880 */
1881 if (pVM->hm.s.vmx.fVpid)
1882 {
1883 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1884 {
1885 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1886 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1887 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1888 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1889 else
1890 {
1891 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1892 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1893 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1894 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1895 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1896 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1897 pVM->hm.s.vmx.fVpid = false;
1898 }
1899 }
1900 else
1901 {
1902 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1903 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1904 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1905 pVM->hm.s.vmx.fVpid = false;
1906 }
1907 }
1908
1909 /*
1910 * Setup the handler for flushing tagged-TLBs.
1911 */
1912 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1913 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1914 else if (pVM->hm.s.fNestedPaging)
1915 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1916 else if (pVM->hm.s.vmx.fVpid)
1917 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1918 else
1919 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1920 return VINF_SUCCESS;
1921}
1922
1923
1924/**
1925 * Sets up pin-based VM-execution controls in the VMCS.
1926 *
1927 * @returns VBox status code.
1928 * @param pVM Pointer to the VM.
1929 * @param pVCpu Pointer to the VMCPU.
1930 */
1931static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1932{
1933 AssertPtr(pVM);
1934 AssertPtr(pVCpu);
1935
1936 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1937 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1938
1939 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1940 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1941 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1942
1943 /* Enable the VMX preemption timer. */
1944 if (pVM->hm.s.vmx.fUsePreemptTimer)
1945 {
1946 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1947 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1948 }
1949
1950 if ((val & zap) != val)
1951 {
1952 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1953 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1954 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1955 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1956 }
1957
1958 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1959 AssertRCReturn(rc, rc);
1960
1961 /* Update VCPU with the currently set pin-based VM-execution controls. */
1962 pVCpu->hm.s.vmx.u32PinCtls = val;
1963 return rc;
1964}
1965
1966
1967/**
1968 * Sets up processor-based VM-execution controls in the VMCS.
1969 *
1970 * @returns VBox status code.
1971 * @param pVM Pointer to the VM.
1972 * @param pVMCPU Pointer to the VMCPU.
1973 */
1974static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1975{
1976 AssertPtr(pVM);
1977 AssertPtr(pVCpu);
1978
1979 int rc = VERR_INTERNAL_ERROR_5;
1980 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1981 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1982
1983 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1984 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1985 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1986 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1987 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1988 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1989 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1990
1991 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1992 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1993 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1994 {
1995 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1996 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1997 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1998 }
1999
2000 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2001 if (!pVM->hm.s.fNestedPaging)
2002 {
2003 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2004 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2005 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2006 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2007 }
2008
2009 /* Use TPR shadowing if supported by the CPU. */
2010 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2011 {
2012 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2013 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2014 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2015 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2016 AssertRCReturn(rc, rc);
2017
2018 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2019 /* CR8 writes causes a VM-exit based on TPR threshold. */
2020 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2021 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2022 }
2023 else
2024 {
2025 /*
2026 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2027 * Set this control only for 64-bit guests.
2028 */
2029 if (pVM->hm.s.fAllow64BitGuests)
2030 {
2031 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2032 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2033 }
2034 }
2035
2036 /* Use MSR-bitmaps if supported by the CPU. */
2037 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2038 {
2039 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2040
2041 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2042 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2043 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2044 AssertRCReturn(rc, rc);
2045
2046 /*
2047 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2048 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
2049 */
2050 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2051 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2052 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2053 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2054 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2055 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2056 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2057 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2058 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2059 }
2060
2061 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2062 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2063 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2064
2065 if ((val & zap) != val)
2066 {
2067 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2068 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2069 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2070 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2071 }
2072
2073 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2074 AssertRCReturn(rc, rc);
2075
2076 /* Update VCPU with the currently set processor-based VM-execution controls. */
2077 pVCpu->hm.s.vmx.u32ProcCtls = val;
2078
2079 /*
2080 * Secondary processor-based VM-execution controls.
2081 */
2082 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2083 {
2084 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2085 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2086
2087 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2088 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2089
2090 if (pVM->hm.s.fNestedPaging)
2091 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2092 else
2093 {
2094 /*
2095 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2096 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2097 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2098 */
2099 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2100 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2101 }
2102
2103 if (pVM->hm.s.vmx.fVpid)
2104 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2105
2106 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2107 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2108
2109 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2110 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2111 * done dynamically. */
2112 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2113 {
2114 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2115 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2116 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2117 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2118 AssertRCReturn(rc, rc);
2119 }
2120
2121 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2122 {
2123 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2124 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2125 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2126 }
2127
2128 if ((val & zap) != val)
2129 {
2130 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2131 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2132 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2133 }
2134
2135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2136 AssertRCReturn(rc, rc);
2137
2138 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2139 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2140 }
2141 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2142 {
2143 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2144 "available\n"));
2145 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2146 }
2147
2148 return VINF_SUCCESS;
2149}
2150
2151
2152/**
2153 * Sets up miscellaneous (everything other than Pin & Processor-based
2154 * VM-execution) control fields in the VMCS.
2155 *
2156 * @returns VBox status code.
2157 * @param pVM Pointer to the VM.
2158 * @param pVCpu Pointer to the VMCPU.
2159 */
2160static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2161{
2162 AssertPtr(pVM);
2163 AssertPtr(pVCpu);
2164
2165 int rc = VERR_GENERAL_FAILURE;
2166
2167 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2168#if 0
2169 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2170 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2171 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2172
2173 /*
2174 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2175 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2176 * We thus use the exception bitmap to control it rather than use both.
2177 */
2178 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2179 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2180
2181 /** @todo Explore possibility of using IO-bitmaps. */
2182 /* All IO & IOIO instructions cause VM-exits. */
2183 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2184 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2185
2186 /* Initialize the MSR-bitmap area. */
2187 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2188 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2189 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2190#endif
2191
2192#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2193 /* Setup MSR autoloading/storing. */
2194 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2195 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2196 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2197 AssertRCReturn(rc, rc);
2198 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2199 AssertRCReturn(rc, rc);
2200
2201 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2202 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2203 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2204 AssertRCReturn(rc, rc);
2205#endif
2206
2207 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2208 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2209 AssertRCReturn(rc, rc);
2210
2211 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2212#if 0
2213 /* Setup debug controls */
2214 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2215 AssertRCReturn(rc, rc);
2216 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2217 AssertRCReturn(rc, rc);
2218#endif
2219
2220 return rc;
2221}
2222
2223
2224/**
2225 * Sets up the initial exception bitmap in the VMCS based on static conditions
2226 * (i.e. conditions that cannot ever change after starting the VM).
2227 *
2228 * @returns VBox status code.
2229 * @param pVM Pointer to the VM.
2230 * @param pVCpu Pointer to the VMCPU.
2231 */
2232static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2233{
2234 AssertPtr(pVM);
2235 AssertPtr(pVCpu);
2236
2237 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2238
2239 uint32_t u32XcptBitmap = 0;
2240
2241 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2242 if (!pVM->hm.s.fNestedPaging)
2243 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2244
2245 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2246 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2247 AssertRCReturn(rc, rc);
2248 return rc;
2249}
2250
2251
2252/**
2253 * Sets up the initial guest-state mask. The guest-state mask is consulted
2254 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2255 * for the nested virtualization case (as it would cause a VM-exit).
2256 *
2257 * @param pVCpu Pointer to the VMCPU.
2258 */
2259static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2260{
2261 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2262 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Does per-VM VT-x initialization.
2269 *
2270 * @returns VBox status code.
2271 * @param pVM Pointer to the VM.
2272 */
2273VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2274{
2275 LogFlowFunc(("pVM=%p\n", pVM));
2276
2277 int rc = hmR0VmxStructsAlloc(pVM);
2278 if (RT_FAILURE(rc))
2279 {
2280 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2281 return rc;
2282 }
2283
2284 return VINF_SUCCESS;
2285}
2286
2287
2288/**
2289 * Does per-VM VT-x termination.
2290 *
2291 * @returns VBox status code.
2292 * @param pVM Pointer to the VM.
2293 */
2294VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2295{
2296 LogFlowFunc(("pVM=%p\n", pVM));
2297
2298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2299 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2300 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2301#endif
2302 hmR0VmxStructsFree(pVM);
2303 return VINF_SUCCESS;
2304}
2305
2306
2307/**
2308 * Sets up the VM for execution under VT-x.
2309 * This function is only called once per-VM during initialization.
2310 *
2311 * @returns VBox status code.
2312 * @param pVM Pointer to the VM.
2313 */
2314VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2315{
2316 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2317 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2318
2319 LogFlowFunc(("pVM=%p\n", pVM));
2320
2321 /*
2322 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2323 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2324 */
2325 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2326 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2327 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2328 || !pVM->hm.s.vmx.pRealModeTSS))
2329 {
2330 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2331 return VERR_INTERNAL_ERROR;
2332 }
2333
2334#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2335 /*
2336 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2337 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2338 */
2339 if ( pVM->hm.s.fAllow64BitGuests
2340 && !HMVMX_IS_64BIT_HOST_MODE())
2341 {
2342 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2343 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2344 }
2345#endif
2346
2347 /* Initialize these always, see hmR3InitFinalizeR0().*/
2348 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2349 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2350
2351 /* Setup the tagged-TLB flush handlers. */
2352 int rc = hmR0VmxSetupTaggedTlb(pVM);
2353 if (RT_FAILURE(rc))
2354 {
2355 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2356 return rc;
2357 }
2358
2359 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2360 {
2361 PVMCPU pVCpu = &pVM->aCpus[i];
2362 AssertPtr(pVCpu);
2363 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2364
2365 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2366 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2367
2368 /* Set revision dword at the beginning of the VMCS structure. */
2369 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2370
2371 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2372 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2373 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2374 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2375
2376 /* Load this VMCS as the current VMCS. */
2377 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2378 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2379 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2380
2381 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2382 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2383 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2384
2385 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2386 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2387 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2388
2389 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2390 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2391 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2392
2393 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2394 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2395 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2396
2397 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2398 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2399 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2400
2401#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2402 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2403 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2404 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2405#endif
2406
2407 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2408 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2409 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2410 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2411
2412 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2413
2414 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2415 }
2416
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2423 * the VMCS.
2424 *
2425 * @returns VBox status code.
2426 * @param pVM Pointer to the VM.
2427 * @param pVCpu Pointer to the VMCPU.
2428 */
2429DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2430{
2431 RTCCUINTREG uReg = ASMGetCR0();
2432 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2433 AssertRCReturn(rc, rc);
2434
2435#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2436 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2437 if (HMVMX_IS_64BIT_HOST_MODE())
2438 {
2439 uint64_t uRegCR3 = HMR0Get64bitCR3();
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2441 }
2442 else
2443#endif
2444 {
2445 uReg = ASMGetCR3();
2446 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2447 }
2448 AssertRCReturn(rc, rc);
2449
2450 uReg = ASMGetCR4();
2451 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2452 AssertRCReturn(rc, rc);
2453 return rc;
2454}
2455
2456
2457#if HC_ARCH_BITS == 64
2458/**
2459 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2460 * requirements. See hmR0VmxSaveHostSegmentRegs().
2461 */
2462# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2463 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2464 { \
2465 bool fValidSelector = true; \
2466 if ((selValue) & X86_SEL_LDT) \
2467 { \
2468 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2469 fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2470 } \
2471 if (fValidSelector) \
2472 { \
2473 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2474 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2475 } \
2476 (selValue) = 0; \
2477 }
2478#endif
2479
2480
2481/**
2482 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2483 * the host-state area in the VMCS.
2484 *
2485 * @returns VBox status code.
2486 * @param pVM Pointer to the VM.
2487 * @param pVCpu Pointer to the VMCPU.
2488 */
2489DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2490{
2491 int rc = VERR_INTERNAL_ERROR_5;
2492
2493 /*
2494 * Host DS, ES, FS and GS segment registers.
2495 */
2496#if HC_ARCH_BITS == 64
2497 RTSEL uSelDS = ASMGetDS();
2498 RTSEL uSelES = ASMGetES();
2499 RTSEL uSelFS = ASMGetFS();
2500 RTSEL uSelGS = ASMGetGS();
2501#else
2502 RTSEL uSelDS = 0;
2503 RTSEL uSelES = 0;
2504 RTSEL uSelFS = 0;
2505 RTSEL uSelGS = 0;
2506#endif
2507
2508 /* Recalculate which host-state bits need to be manually restored. */
2509 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2510
2511 /*
2512 * Host CS and SS segment registers.
2513 */
2514#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2515 RTSEL uSelCS;
2516 RTSEL uSelSS;
2517 if (HMVMX_IS_64BIT_HOST_MODE())
2518 {
2519 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2520 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2521 }
2522 else
2523 {
2524 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2525 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2526 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2527 }
2528#else
2529 RTSEL uSelCS = ASMGetCS();
2530 RTSEL uSelSS = ASMGetSS();
2531#endif
2532
2533 /*
2534 * Host TR segment register.
2535 */
2536 RTSEL uSelTR = ASMGetTR();
2537
2538#if HC_ARCH_BITS == 64
2539 /*
2540 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2541 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2542 */
2543 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2544 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2545 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2546 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2547# undef VMXLOCAL_ADJUST_HOST_SEG
2548#endif
2549
2550 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2551 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2552 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2553 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2554 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2555 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2556 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2557 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2558 Assert(uSelCS);
2559 Assert(uSelTR);
2560
2561 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2562#if 0
2563 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2564 Assert(uSelSS != 0);
2565#endif
2566
2567 /* Write these host selector fields into the host-state area in the VMCS. */
2568 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2570#if HC_ARCH_BITS == 64
2571 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2575#endif
2576 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2577
2578 /*
2579 * Host GDTR and IDTR.
2580 */
2581 RTGDTR Gdtr;
2582 RT_ZERO(Gdtr);
2583#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2584 if (HMVMX_IS_64BIT_HOST_MODE())
2585 {
2586 X86XDTR64 Gdtr64;
2587 X86XDTR64 Idtr64;
2588 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2589 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2591
2592 Gdtr.cbGdt = Gdtr64.cb;
2593 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2594 }
2595 else
2596#endif
2597 {
2598 RTIDTR Idtr;
2599 ASMGetGDTR(&Gdtr);
2600 ASMGetIDTR(&Idtr);
2601 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2602 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2603
2604#if HC_ARCH_BITS == 64
2605 /*
2606 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2607 * maximum limit (0xffff) on every VM-exit.
2608 */
2609 if (Gdtr.cbGdt != 0xffff)
2610 {
2611 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2612 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2613 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2614 }
2615
2616 /*
2617 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2618 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2619 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2620 */
2621 if (Idtr.cbIdt < 0x0fff)
2622 {
2623 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2624 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2625 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2626 }
2627#endif
2628 }
2629
2630 /*
2631 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2632 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2633 */
2634 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2635 {
2636 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2637 return VERR_VMX_INVALID_HOST_STATE;
2638 }
2639
2640 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2641#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2642 if (HMVMX_IS_64BIT_HOST_MODE())
2643 {
2644 /* We need the 64-bit TR base for hybrid darwin. */
2645 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2646 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2647 }
2648 else
2649#endif
2650 {
2651 uintptr_t uTRBase;
2652#if HC_ARCH_BITS == 64
2653 uTRBase = X86DESC64_BASE(pDesc);
2654
2655 /*
2656 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2657 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2658 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2659 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2660 *
2661 * [1] See Intel spec. 3.5 "System Descriptor Types".
2662 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2663 */
2664 Assert(pDesc->System.u4Type == 11);
2665 if ( pDesc->System.u16LimitLow != 0x67
2666 || pDesc->System.u4LimitHigh)
2667 {
2668 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2669 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2670
2671 /* Store the GDTR here as we need it while restoring TR. */
2672 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2673 }
2674#else
2675 uTRBase = X86DESC_BASE(pDesc);
2676#endif
2677 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2678 }
2679 AssertRCReturn(rc, rc);
2680
2681 /*
2682 * Host FS base and GS base.
2683 */
2684#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2685 if (HMVMX_IS_64BIT_HOST_MODE())
2686 {
2687 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2688 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2689 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2690 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2691
2692# if HC_ARCH_BITS == 64
2693 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2694 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2695 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2696 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2697 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2698# endif
2699 }
2700#endif
2701 return rc;
2702}
2703
2704
2705/**
2706 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2707 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2708 * the host after every successful VM exit.
2709 *
2710 * @returns VBox status code.
2711 * @param pVM Pointer to the VM.
2712 * @param pVCpu Pointer to the VMCPU.
2713 */
2714DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2715{
2716 AssertPtr(pVCpu);
2717 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2718
2719 int rc = VINF_SUCCESS;
2720#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2721 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2722 uint32_t cHostMsrs = 0;
2723 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2724
2725 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2726 {
2727 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2728
2729# if HC_ARCH_BITS == 64
2730 /* Paranoia. 64-bit code requires these bits to be set always. */
2731 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2732
2733 /*
2734 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2735 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2736 * some reason (e.g. allow transparent reads) we would activate the code below.
2737 */
2738# if 0
2739 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2740 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2741 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2742 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2743 if (CPUMIsGuestInLongMode(pVCpu))
2744 {
2745 uint64_t u64GuestEfer;
2746 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2747 AssertRC(rc);
2748
2749 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2750 {
2751 pHostMsr->u32Msr = MSR_K6_EFER;
2752 pHostMsr->u32Reserved = 0;
2753 pHostMsr->u64Value = u64HostEfer;
2754 pHostMsr++; cHostMsrs++;
2755 }
2756 }
2757# endif
2758# else /* HC_ARCH_BITS != 64 */
2759 pHostMsr->u32Msr = MSR_K6_EFER;
2760 pHostMsr->u32Reserved = 0;
2761# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2762 if (CPUMIsGuestInLongMode(pVCpu))
2763 {
2764 /* Must match the EFER value in our 64 bits switcher. */
2765 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2766 }
2767 else
2768# endif
2769 pHostMsr->u64Value = u64HostEfer;
2770 pHostMsr++; cHostMsrs++;
2771# endif /* HC_ARCH_BITS == 64 */
2772 }
2773
2774# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2775 if (HMVMX_IS_64BIT_HOST_MODE())
2776 {
2777 pHostMsr->u32Msr = MSR_K6_STAR;
2778 pHostMsr->u32Reserved = 0;
2779 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2780 pHostMsr++; cHostMsrs++;
2781 pHostMsr->u32Msr = MSR_K8_LSTAR;
2782 pHostMsr->u32Reserved = 0;
2783 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2784 pHostMsr++; cHostMsrs++;
2785 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2786 pHostMsr->u32Reserved = 0;
2787 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2788 pHostMsr++; cHostMsrs++;
2789 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2790 pHostMsr->u32Reserved = 0;
2791 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2792 pHostMsr++; cHostMsrs++;
2793 }
2794# endif
2795
2796 /* Host TSC AUX MSR must be restored since we always load/store guest TSC AUX MSR. */
2797 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2798 {
2799 pHostMsr->u32Msr = MSR_K8_TSC_AUX;
2800 pHostMsr->u32Reserved = 0;
2801 pHostMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
2802 pHostMsr++; cHostMsrs++;
2803 }
2804
2805 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2806 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2807 {
2808 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2809 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2810 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2811 }
2812
2813 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2814#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2815
2816 /*
2817 * Host Sysenter MSRs.
2818 */
2819 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2820 AssertRCReturn(rc, rc);
2821#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2822 if (HMVMX_IS_64BIT_HOST_MODE())
2823 {
2824 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2825 AssertRCReturn(rc, rc);
2826 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2827 }
2828 else
2829 {
2830 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2831 AssertRCReturn(rc, rc);
2832 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2833 }
2834#elif HC_ARCH_BITS == 32
2835 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2836 AssertRCReturn(rc, rc);
2837 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2838#else
2839 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2840 AssertRCReturn(rc, rc);
2841 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2842#endif
2843 AssertRCReturn(rc, rc);
2844
2845 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2846 * hmR0VmxSetupExitCtls() !! */
2847 return rc;
2848}
2849
2850
2851/**
2852 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2853 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2854 * controls".
2855 *
2856 * @returns VBox status code.
2857 * @param pVCpu Pointer to the VMCPU.
2858 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2859 * out-of-sync. Make sure to update the required fields
2860 * before using them.
2861 *
2862 * @remarks No-long-jump zone!!!
2863 */
2864DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2865{
2866 int rc = VINF_SUCCESS;
2867 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2868 {
2869 PVM pVM = pVCpu->CTX_SUFF(pVM);
2870 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2871 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2872
2873 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2874 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2875
2876 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2877 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2878 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2879 else
2880 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2881
2882 /*
2883 * The following should -not- be set (since we're not in SMM mode):
2884 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2885 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2886 */
2887
2888 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2889 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2890 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2891
2892 if ((val & zap) != val)
2893 {
2894 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2895 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2896 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2897 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2898 }
2899
2900 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2901 AssertRCReturn(rc, rc);
2902
2903 /* Update VCPU with the currently set VM-exit controls. */
2904 pVCpu->hm.s.vmx.u32EntryCtls = val;
2905 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2906 }
2907 return rc;
2908}
2909
2910
2911/**
2912 * Sets up the VM-exit controls in the VMCS.
2913 *
2914 * @returns VBox status code.
2915 * @param pVM Pointer to the VM.
2916 * @param pVCpu Pointer to the VMCPU.
2917 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2918 * out-of-sync. Make sure to update the required fields
2919 * before using them.
2920 *
2921 * @remarks requires EFER.
2922 */
2923DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2924{
2925 int rc = VINF_SUCCESS;
2926 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2927 {
2928 PVM pVM = pVCpu->CTX_SUFF(pVM);
2929 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2930 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2931
2932 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2933 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2934
2935 /*
2936 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2937 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2938 */
2939#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2940 if (HMVMX_IS_64BIT_HOST_MODE())
2941 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2942 else
2943 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2944#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2945 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2946 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2947 else
2948 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2949#endif
2950
2951 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2952 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2953
2954 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2955 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2956 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2957 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2958 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2959
2960 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2961 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2962
2963 if ((val & zap) != val)
2964 {
2965 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2966 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2967 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2968 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2969 }
2970
2971 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2972 AssertRCReturn(rc, rc);
2973
2974 /* Update VCPU with the currently set VM-exit controls. */
2975 pVCpu->hm.s.vmx.u32ExitCtls = val;
2976 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2977 }
2978 return rc;
2979}
2980
2981
2982/**
2983 * Loads the guest APIC and related state.
2984 *
2985 * @returns VBox status code.
2986 * @param pVM Pointer to the VM.
2987 * @param pVCpu Pointer to the VMCPU.
2988 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2989 * out-of-sync. Make sure to update the required fields
2990 * before using them.
2991 */
2992DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2993{
2994 int rc = VINF_SUCCESS;
2995 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2996 {
2997 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2998 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2999 {
3000 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3001
3002 bool fPendingIntr = false;
3003 uint8_t u8Tpr = 0;
3004 uint8_t u8PendingIntr = 0;
3005 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3006 AssertRCReturn(rc, rc);
3007
3008 /*
3009 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3010 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3011 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3012 * the interrupt when we VM-exit for other reasons.
3013 */
3014 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3015 uint32_t u32TprThreshold = 0;
3016 if (fPendingIntr)
3017 {
3018 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3019 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3020 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3021 if (u8PendingPriority <= u8TprPriority)
3022 u32TprThreshold = u8PendingPriority;
3023 else
3024 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3025 }
3026 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3027
3028 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3029 AssertRCReturn(rc, rc);
3030 }
3031
3032 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3033 }
3034 return rc;
3035}
3036
3037
3038/**
3039 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3040 *
3041 * @returns Guest's interruptibility-state.
3042 * @param pVCpu Pointer to the VMCPU.
3043 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3044 * out-of-sync. Make sure to update the required fields
3045 * before using them.
3046 *
3047 * @remarks No-long-jump zone!!!
3048 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3049 */
3050DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3051{
3052 /*
3053 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3054 * inhibit interrupts or clear any existing interrupt-inhibition.
3055 */
3056 uint32_t uIntrState = 0;
3057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3058 {
3059 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3060 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
3061 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3062 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3063 {
3064 /*
3065 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3066 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3067 */
3068 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3069 }
3070 else if (pMixedCtx->eflags.Bits.u1IF)
3071 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3072 else
3073 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3074 }
3075 return uIntrState;
3076}
3077
3078
3079/**
3080 * Loads the guest's interruptibility-state into the guest-state area in the
3081 * VMCS.
3082 *
3083 * @returns VBox status code.
3084 * @param pVCpu Pointer to the VMCPU.
3085 * @param uIntrState The interruptibility-state to set.
3086 */
3087static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3088{
3089 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3090 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3091 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3092 AssertRCReturn(rc, rc);
3093 return rc;
3094}
3095
3096
3097/**
3098 * Loads the guest's RIP into the guest-state area in the VMCS.
3099 *
3100 * @returns VBox status code.
3101 * @param pVCpu Pointer to the VMCPU.
3102 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3103 * out-of-sync. Make sure to update the required fields
3104 * before using them.
3105 *
3106 * @remarks No-long-jump zone!!!
3107 */
3108static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3109{
3110 int rc = VINF_SUCCESS;
3111 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3112 {
3113 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3114 AssertRCReturn(rc, rc);
3115
3116 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3117 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3118 }
3119 return rc;
3120}
3121
3122
3123/**
3124 * Loads the guest's RSP into the guest-state area in the VMCS.
3125 *
3126 * @returns VBox status code.
3127 * @param pVCpu Pointer to the VMCPU.
3128 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3129 * out-of-sync. Make sure to update the required fields
3130 * before using them.
3131 *
3132 * @remarks No-long-jump zone!!!
3133 */
3134static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3135{
3136 int rc = VINF_SUCCESS;
3137 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3138 {
3139 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3140 AssertRCReturn(rc, rc);
3141
3142 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3143 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3144 }
3145 return rc;
3146}
3147
3148
3149/**
3150 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3151 *
3152 * @returns VBox status code.
3153 * @param pVCpu Pointer to the VMCPU.
3154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3155 * out-of-sync. Make sure to update the required fields
3156 * before using them.
3157 *
3158 * @remarks No-long-jump zone!!!
3159 */
3160static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3161{
3162 int rc = VINF_SUCCESS;
3163 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3164 {
3165 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3166 Let us assert it as such and use 32-bit VMWRITE. */
3167 Assert(!(pMixedCtx->rflags.u64 >> 32));
3168 X86EFLAGS Eflags = pMixedCtx->eflags;
3169 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3170 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3171
3172 /*
3173 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3174 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3175 */
3176 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3177 {
3178 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3179 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3180 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3181 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3182 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3183 }
3184
3185 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3186 AssertRCReturn(rc, rc);
3187
3188 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3189 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3190 }
3191 return rc;
3192}
3193
3194
3195/**
3196 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3197 *
3198 * @returns VBox status code.
3199 * @param pVCpu Pointer to the VMCPU.
3200 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201 * out-of-sync. Make sure to update the required fields
3202 * before using them.
3203 *
3204 * @remarks No-long-jump zone!!!
3205 */
3206DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3207{
3208 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3209 AssertRCReturn(rc, rc);
3210 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3211 AssertRCReturn(rc, rc);
3212 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3213 AssertRCReturn(rc, rc);
3214 return rc;
3215}
3216
3217
3218/**
3219 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3220 * CR0 is partially shared with the host and we have to consider the FPU bits.
3221 *
3222 * @returns VBox status code.
3223 * @param pVM Pointer to the VM.
3224 * @param pVCpu Pointer to the VMCPU.
3225 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3226 * out-of-sync. Make sure to update the required fields
3227 * before using them.
3228 *
3229 * @remarks No-long-jump zone!!!
3230 */
3231static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3232{
3233 /*
3234 * Guest CR0.
3235 * Guest FPU.
3236 */
3237 int rc = VINF_SUCCESS;
3238 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3239 {
3240 Assert(!(pMixedCtx->cr0 >> 32));
3241 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3242 PVM pVM = pVCpu->CTX_SUFF(pVM);
3243
3244 /* The guest's view (read access) of its CR0 is unblemished. */
3245 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3246 AssertRCReturn(rc, rc);
3247 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3248
3249 /* Setup VT-x's view of the guest CR0. */
3250 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3251 if (pVM->hm.s.fNestedPaging)
3252 {
3253 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3254 {
3255 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3256 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3257 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3258 }
3259 else
3260 {
3261 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3262 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3263 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3264 }
3265
3266 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3267 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3268 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3269
3270 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3271 AssertRCReturn(rc, rc);
3272 }
3273 else
3274 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3275
3276 /*
3277 * Guest FPU bits.
3278 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3279 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3280 */
3281 u32GuestCR0 |= X86_CR0_NE;
3282 bool fInterceptNM = false;
3283 if (CPUMIsGuestFPUStateActive(pVCpu))
3284 {
3285 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3286 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3287 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3288 }
3289 else
3290 {
3291 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3292 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3293 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3294 }
3295
3296 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3297 bool fInterceptMF = false;
3298 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3299 fInterceptMF = true;
3300
3301 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3302 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3303 {
3304 Assert(PDMVmmDevHeapIsEnabled(pVM));
3305 Assert(pVM->hm.s.vmx.pRealModeTSS);
3306 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3307 fInterceptNM = true;
3308 fInterceptMF = true;
3309 }
3310 else
3311 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3312
3313 if (fInterceptNM)
3314 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3315 else
3316 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3317
3318 if (fInterceptMF)
3319 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3320 else
3321 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3322
3323 /* Additional intercepts for debugging, define these yourself explicitly. */
3324#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3325 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3326 | RT_BIT(X86_XCPT_BP)
3327 | RT_BIT(X86_XCPT_DB)
3328 | RT_BIT(X86_XCPT_DE)
3329 | RT_BIT(X86_XCPT_NM)
3330 | RT_BIT(X86_XCPT_UD)
3331 | RT_BIT(X86_XCPT_NP)
3332 | RT_BIT(X86_XCPT_SS)
3333 | RT_BIT(X86_XCPT_GP)
3334 | RT_BIT(X86_XCPT_PF)
3335 | RT_BIT(X86_XCPT_MF)
3336 ;
3337#elif defined(HMVMX_ALWAYS_TRAP_PF)
3338 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3339#endif
3340
3341 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3342
3343 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3344 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3345 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3346 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3347 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3348 else
3349 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3350
3351 u32GuestCR0 |= uSetCR0;
3352 u32GuestCR0 &= uZapCR0;
3353 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3354
3355 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3356 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3357 AssertRCReturn(rc, rc);
3358 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3359 AssertRCReturn(rc, rc);
3360 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3361
3362 /*
3363 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3364 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3365 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3366 */
3367 uint32_t u32CR0Mask = 0;
3368 u32CR0Mask = X86_CR0_PE
3369 | X86_CR0_NE
3370 | X86_CR0_WP
3371 | X86_CR0_PG
3372 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3373 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3374 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3375
3376 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3377 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3378 * and @bugref{6944}. */
3379#if 0
3380 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3381 u32CR0Mask &= ~X86_CR0_PE;
3382#endif
3383 if (pVM->hm.s.fNestedPaging)
3384 u32CR0Mask &= ~X86_CR0_WP;
3385
3386 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3387 if (fInterceptNM)
3388 {
3389 u32CR0Mask |= X86_CR0_TS
3390 | X86_CR0_MP;
3391 }
3392
3393 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3394 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3395 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3396 AssertRCReturn(rc, rc);
3397 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3398
3399 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3400 }
3401 return rc;
3402}
3403
3404
3405/**
3406 * Loads the guest control registers (CR3, CR4) into the guest-state area
3407 * in the VMCS.
3408 *
3409 * @returns VBox status code.
3410 * @param pVM Pointer to the VM.
3411 * @param pVCpu Pointer to the VMCPU.
3412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3413 * out-of-sync. Make sure to update the required fields
3414 * before using them.
3415 *
3416 * @remarks No-long-jump zone!!!
3417 */
3418static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3419{
3420 int rc = VINF_SUCCESS;
3421 PVM pVM = pVCpu->CTX_SUFF(pVM);
3422
3423 /*
3424 * Guest CR2.
3425 * It's always loaded in the assembler code. Nothing to do here.
3426 */
3427
3428 /*
3429 * Guest CR3.
3430 */
3431 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3432 {
3433 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3434 if (pVM->hm.s.fNestedPaging)
3435 {
3436 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3437
3438 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3439 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3440 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3441 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3442
3443 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3444 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3445 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3446
3447 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3448 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3449 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3450 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3451
3452 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3453 AssertRCReturn(rc, rc);
3454 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3455
3456 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3457 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3458 {
3459 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3460 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3461 {
3462 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3463 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3464 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3465 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3466 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3467 }
3468
3469 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3470 have Unrestricted Execution to handle the guest when it's not using paging. */
3471 GCPhysGuestCR3 = pMixedCtx->cr3;
3472 }
3473 else
3474 {
3475 /*
3476 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3477 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3478 * EPT takes care of translating it to host-physical addresses.
3479 */
3480 RTGCPHYS GCPhys;
3481 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3482 Assert(PDMVmmDevHeapIsEnabled(pVM));
3483
3484 /* We obtain it here every time as the guest could have relocated this PCI region. */
3485 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3486 AssertRCReturn(rc, rc);
3487
3488 GCPhysGuestCR3 = GCPhys;
3489 }
3490
3491 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3492 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3493 }
3494 else
3495 {
3496 /* Non-nested paging case, just use the hypervisor's CR3. */
3497 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3498
3499 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3500 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3501 }
3502 AssertRCReturn(rc, rc);
3503
3504 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3505 }
3506
3507 /*
3508 * Guest CR4.
3509 */
3510 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3511 {
3512 Assert(!(pMixedCtx->cr4 >> 32));
3513 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3514
3515 /* The guest's view of its CR4 is unblemished. */
3516 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3517 AssertRCReturn(rc, rc);
3518 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3519
3520 /* Setup VT-x's view of the guest CR4. */
3521 /*
3522 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3523 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3524 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3525 */
3526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3527 {
3528 Assert(pVM->hm.s.vmx.pRealModeTSS);
3529 Assert(PDMVmmDevHeapIsEnabled(pVM));
3530 u32GuestCR4 &= ~X86_CR4_VME;
3531 }
3532
3533 if (pVM->hm.s.fNestedPaging)
3534 {
3535 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3536 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3537 {
3538 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3539 u32GuestCR4 |= X86_CR4_PSE;
3540 /* Our identity mapping is a 32 bits page directory. */
3541 u32GuestCR4 &= ~X86_CR4_PAE;
3542 }
3543 /* else use guest CR4.*/
3544 }
3545 else
3546 {
3547 /*
3548 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3549 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3550 */
3551 switch (pVCpu->hm.s.enmShadowMode)
3552 {
3553 case PGMMODE_REAL: /* Real-mode. */
3554 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3555 case PGMMODE_32_BIT: /* 32-bit paging. */
3556 {
3557 u32GuestCR4 &= ~X86_CR4_PAE;
3558 break;
3559 }
3560
3561 case PGMMODE_PAE: /* PAE paging. */
3562 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3563 {
3564 u32GuestCR4 |= X86_CR4_PAE;
3565 break;
3566 }
3567
3568 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3569 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3570#ifdef VBOX_ENABLE_64_BITS_GUESTS
3571 break;
3572#endif
3573 default:
3574 AssertFailed();
3575 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3576 }
3577 }
3578
3579 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3580 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3581 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3582 u32GuestCR4 |= uSetCR4;
3583 u32GuestCR4 &= uZapCR4;
3584
3585 /* Write VT-x's view of the guest CR4 into the VMCS. */
3586 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3587 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3588 AssertRCReturn(rc, rc);
3589
3590 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3591 uint32_t u32CR4Mask = 0;
3592 u32CR4Mask = X86_CR4_VME
3593 | X86_CR4_PAE
3594 | X86_CR4_PGE
3595 | X86_CR4_PSE
3596 | X86_CR4_VMXE;
3597 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3598 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3599 AssertRCReturn(rc, rc);
3600
3601 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3602 }
3603 return rc;
3604}
3605
3606
3607/**
3608 * Loads the guest debug registers into the guest-state area in the VMCS.
3609 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3610 *
3611 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu Pointer to the VMCPU.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3624 return VINF_SUCCESS;
3625
3626#ifdef VBOX_STRICT
3627 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3628 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3629 {
3630 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3631 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3632 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3633 }
3634#endif
3635
3636 int rc;
3637 PVM pVM = pVCpu->CTX_SUFF(pVM);
3638 bool fInterceptDB = false;
3639 bool fInterceptMovDRx = false;
3640 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3641 {
3642 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3643 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3644 {
3645 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3646 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3647 AssertRCReturn(rc, rc);
3648 Assert(fInterceptDB == false);
3649 }
3650 else
3651 {
3652 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3653 pVCpu->hm.s.fClearTrapFlag = true;
3654 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3655 fInterceptDB = true;
3656 }
3657 }
3658
3659 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3660 {
3661 /*
3662 * Use the combined guest and host DRx values found in the hypervisor
3663 * register set because the debugger has breakpoints active or someone
3664 * is single stepping on the host side without a monitor trap flag.
3665 *
3666 * Note! DBGF expects a clean DR6 state before executing guest code.
3667 */
3668#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3669 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3670 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3671 {
3672 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3673 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3674 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3675 }
3676 else
3677#endif
3678 if (!CPUMIsHyperDebugStateActive(pVCpu))
3679 {
3680 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3681 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3682 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3683 }
3684
3685 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3686 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3687 AssertRCReturn(rc, rc);
3688
3689 pVCpu->hm.s.fUsingHyperDR7 = true;
3690 fInterceptDB = true;
3691 fInterceptMovDRx = true;
3692 }
3693 else
3694 {
3695 /*
3696 * If the guest has enabled debug registers, we need to load them prior to
3697 * executing guest code so they'll trigger at the right time.
3698 */
3699 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3700 {
3701#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3702 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3703 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3704 {
3705 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3706 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3707 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3708 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3709 }
3710 else
3711#endif
3712 if (CPUMIsGuestDebugStateActive(pVCpu))
3713 {
3714 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3715 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3716 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3717 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3718 }
3719 }
3720 /*
3721 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3722 * must intercept #DB in order to maintain a correct DR6 guest value.
3723 */
3724#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3725 else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3726 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3727 || !CPUMIsGuestDebugStateActive(pVCpu))
3728#else
3729 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3730#endif
3731 {
3732 fInterceptMovDRx = true;
3733 fInterceptDB = true;
3734 }
3735
3736 /* Update guest DR7. */
3737 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3738 AssertRCReturn(rc, rc);
3739
3740 pVCpu->hm.s.fUsingHyperDR7 = false;
3741 }
3742
3743 /*
3744 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3745 */
3746 if (fInterceptDB)
3747 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3748 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3749 {
3750#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3751 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3752#endif
3753 }
3754 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3755 AssertRCReturn(rc, rc);
3756
3757 /*
3758 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3759 */
3760 if (fInterceptMovDRx)
3761 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3762 else
3763 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3764 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3765 AssertRCReturn(rc, rc);
3766
3767 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3768 return VINF_SUCCESS;
3769}
3770
3771
3772#ifdef VBOX_STRICT
3773/**
3774 * Strict function to validate segment registers.
3775 *
3776 * @remarks ASSUMES CR0 is up to date.
3777 */
3778static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3779{
3780 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3781 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3782 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3783 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3784 && ( !CPUMIsGuestInRealModeEx(pCtx)
3785 && !CPUMIsGuestInV86ModeEx(pCtx)))
3786 {
3787 /* Protected mode checks */
3788 /* CS */
3789 Assert(pCtx->cs.Attr.n.u1Present);
3790 Assert(!(pCtx->cs.Attr.u & 0xf00));
3791 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3792 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3793 || !(pCtx->cs.Attr.n.u1Granularity));
3794 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3795 || (pCtx->cs.Attr.n.u1Granularity));
3796 /* CS cannot be loaded with NULL in protected mode. */
3797 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3798 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3799 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3800 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3801 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3802 else
3803 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3804 /* SS */
3805 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3806 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3807 if ( !(pCtx->cr0 & X86_CR0_PE)
3808 || pCtx->cs.Attr.n.u4Type == 3)
3809 {
3810 Assert(!pCtx->ss.Attr.n.u2Dpl);
3811 }
3812 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3813 {
3814 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3815 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3816 Assert(pCtx->ss.Attr.n.u1Present);
3817 Assert(!(pCtx->ss.Attr.u & 0xf00));
3818 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3819 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3820 || !(pCtx->ss.Attr.n.u1Granularity));
3821 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3822 || (pCtx->ss.Attr.n.u1Granularity));
3823 }
3824 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3825 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3826 {
3827 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3828 Assert(pCtx->ds.Attr.n.u1Present);
3829 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3830 Assert(!(pCtx->ds.Attr.u & 0xf00));
3831 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3832 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3833 || !(pCtx->ds.Attr.n.u1Granularity));
3834 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3835 || (pCtx->ds.Attr.n.u1Granularity));
3836 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3837 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3838 }
3839 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3840 {
3841 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3842 Assert(pCtx->es.Attr.n.u1Present);
3843 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3844 Assert(!(pCtx->es.Attr.u & 0xf00));
3845 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3846 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3847 || !(pCtx->es.Attr.n.u1Granularity));
3848 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3849 || (pCtx->es.Attr.n.u1Granularity));
3850 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3851 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3852 }
3853 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3854 {
3855 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3856 Assert(pCtx->fs.Attr.n.u1Present);
3857 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3858 Assert(!(pCtx->fs.Attr.u & 0xf00));
3859 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3860 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3861 || !(pCtx->fs.Attr.n.u1Granularity));
3862 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3863 || (pCtx->fs.Attr.n.u1Granularity));
3864 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3865 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3866 }
3867 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3868 {
3869 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3870 Assert(pCtx->gs.Attr.n.u1Present);
3871 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3872 Assert(!(pCtx->gs.Attr.u & 0xf00));
3873 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3874 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3875 || !(pCtx->gs.Attr.n.u1Granularity));
3876 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3877 || (pCtx->gs.Attr.n.u1Granularity));
3878 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3879 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3880 }
3881 /* 64-bit capable CPUs. */
3882# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3883 Assert(!(pCtx->cs.u64Base >> 32));
3884 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3885 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3886 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3887# endif
3888 }
3889 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3890 || ( CPUMIsGuestInRealModeEx(pCtx)
3891 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3892 {
3893 /* Real and v86 mode checks. */
3894 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3895 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3896 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3897 {
3898 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3899 }
3900 else
3901 {
3902 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3903 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3904 }
3905
3906 /* CS */
3907 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3908 Assert(pCtx->cs.u32Limit == 0xffff);
3909 Assert(u32CSAttr == 0xf3);
3910 /* SS */
3911 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3912 Assert(pCtx->ss.u32Limit == 0xffff);
3913 Assert(u32SSAttr == 0xf3);
3914 /* DS */
3915 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3916 Assert(pCtx->ds.u32Limit == 0xffff);
3917 Assert(u32DSAttr == 0xf3);
3918 /* ES */
3919 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3920 Assert(pCtx->es.u32Limit == 0xffff);
3921 Assert(u32ESAttr == 0xf3);
3922 /* FS */
3923 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3924 Assert(pCtx->fs.u32Limit == 0xffff);
3925 Assert(u32FSAttr == 0xf3);
3926 /* GS */
3927 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3928 Assert(pCtx->gs.u32Limit == 0xffff);
3929 Assert(u32GSAttr == 0xf3);
3930 /* 64-bit capable CPUs. */
3931# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3932 Assert(!(pCtx->cs.u64Base >> 32));
3933 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3934 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3935 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3936# endif
3937 }
3938}
3939#endif /* VBOX_STRICT */
3940
3941
3942/**
3943 * Writes a guest segment register into the guest-state area in the VMCS.
3944 *
3945 * @returns VBox status code.
3946 * @param pVCpu Pointer to the VMCPU.
3947 * @param idxSel Index of the selector in the VMCS.
3948 * @param idxLimit Index of the segment limit in the VMCS.
3949 * @param idxBase Index of the segment base in the VMCS.
3950 * @param idxAccess Index of the access rights of the segment in the VMCS.
3951 * @param pSelReg Pointer to the segment selector.
3952 * @param pCtx Pointer to the guest-CPU context.
3953 *
3954 * @remarks No-long-jump zone!!!
3955 */
3956static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3957 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3958{
3959 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3960 AssertRCReturn(rc, rc);
3961 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3962 AssertRCReturn(rc, rc);
3963 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3964 AssertRCReturn(rc, rc);
3965
3966 uint32_t u32Access = pSelReg->Attr.u;
3967 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3968 {
3969 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3970 u32Access = 0xf3;
3971 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3972 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3973 }
3974 else
3975 {
3976 /*
3977 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3978 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3979 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3980 * loaded in protected-mode have their attribute as 0.
3981 */
3982 if (!u32Access)
3983 u32Access = X86DESCATTR_UNUSABLE;
3984 }
3985
3986 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3987 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3988 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3989
3990 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3991 AssertRCReturn(rc, rc);
3992 return rc;
3993}
3994
3995
3996/**
3997 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3998 * into the guest-state area in the VMCS.
3999 *
4000 * @returns VBox status code.
4001 * @param pVM Pointer to the VM.
4002 * @param pVCPU Pointer to the VMCPU.
4003 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4004 * out-of-sync. Make sure to update the required fields
4005 * before using them.
4006 *
4007 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4008 * @remarks No-long-jump zone!!!
4009 */
4010static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4011{
4012 int rc = VERR_INTERNAL_ERROR_5;
4013 PVM pVM = pVCpu->CTX_SUFF(pVM);
4014
4015 /*
4016 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4017 */
4018 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4019 {
4020 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4021 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4022 {
4023 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4024 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4025 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4026 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4027 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4028 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4029 }
4030
4031#ifdef VBOX_WITH_REM
4032 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4033 {
4034 Assert(pVM->hm.s.vmx.pRealModeTSS);
4035 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4036 if ( pVCpu->hm.s.vmx.fWasInRealMode
4037 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4038 {
4039 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4040 in real-mode (e.g. OpenBSD 4.0) */
4041 REMFlushTBs(pVM);
4042 Log4(("Load: Switch to protected mode detected!\n"));
4043 pVCpu->hm.s.vmx.fWasInRealMode = false;
4044 }
4045 }
4046#endif
4047 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4048 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
4049 AssertRCReturn(rc, rc);
4050 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4051 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
4052 AssertRCReturn(rc, rc);
4053 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4054 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
4055 AssertRCReturn(rc, rc);
4056 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4057 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
4058 AssertRCReturn(rc, rc);
4059 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4060 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
4061 AssertRCReturn(rc, rc);
4062 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4063 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
4064 AssertRCReturn(rc, rc);
4065
4066#ifdef VBOX_STRICT
4067 /* Validate. */
4068 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4069#endif
4070
4071 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4072 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4073 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4074 }
4075
4076 /*
4077 * Guest TR.
4078 */
4079 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4080 {
4081 /*
4082 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4083 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4084 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4085 */
4086 uint16_t u16Sel = 0;
4087 uint32_t u32Limit = 0;
4088 uint64_t u64Base = 0;
4089 uint32_t u32AccessRights = 0;
4090
4091 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4092 {
4093 u16Sel = pMixedCtx->tr.Sel;
4094 u32Limit = pMixedCtx->tr.u32Limit;
4095 u64Base = pMixedCtx->tr.u64Base;
4096 u32AccessRights = pMixedCtx->tr.Attr.u;
4097 }
4098 else
4099 {
4100 Assert(pVM->hm.s.vmx.pRealModeTSS);
4101 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4102
4103 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4104 RTGCPHYS GCPhys;
4105 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4106 AssertRCReturn(rc, rc);
4107
4108 X86DESCATTR DescAttr;
4109 DescAttr.u = 0;
4110 DescAttr.n.u1Present = 1;
4111 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4112
4113 u16Sel = 0;
4114 u32Limit = HM_VTX_TSS_SIZE;
4115 u64Base = GCPhys; /* in real-mode phys = virt. */
4116 u32AccessRights = DescAttr.u;
4117 }
4118
4119 /* Validate. */
4120 Assert(!(u16Sel & RT_BIT(2)));
4121 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4122 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4123 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4124 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4125 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4126 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4127 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4128 Assert( (u32Limit & 0xfff) == 0xfff
4129 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4130 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4131 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4132
4133 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4134 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4135 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4136 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4137
4138 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4139 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4140 }
4141
4142 /*
4143 * Guest GDTR.
4144 */
4145 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4146 {
4147 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4148 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4149
4150 /* Validate. */
4151 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4152
4153 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4154 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4155 }
4156
4157 /*
4158 * Guest LDTR.
4159 */
4160 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4161 {
4162 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4163 uint32_t u32Access = 0;
4164 if (!pMixedCtx->ldtr.Attr.u)
4165 u32Access = X86DESCATTR_UNUSABLE;
4166 else
4167 u32Access = pMixedCtx->ldtr.Attr.u;
4168
4169 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4170 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4171 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4172 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4173
4174 /* Validate. */
4175 if (!(u32Access & X86DESCATTR_UNUSABLE))
4176 {
4177 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4178 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4179 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4180 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4181 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4182 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4183 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4184 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4185 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4186 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4187 }
4188
4189 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4190 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4191 }
4192
4193 /*
4194 * Guest IDTR.
4195 */
4196 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4197 {
4198 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4199 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4200
4201 /* Validate. */
4202 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4203
4204 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4205 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4206 }
4207
4208 return VINF_SUCCESS;
4209}
4210
4211
4212/**
4213 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4214 * areas. These MSRs will automatically be loaded to the host CPU on every
4215 * successful VM entry and stored from the host CPU on every successful VM exit.
4216 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4217 *
4218 * @returns VBox status code.
4219 * @param pVCpu Pointer to the VMCPU.
4220 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4221 * out-of-sync. Make sure to update the required fields
4222 * before using them.
4223 *
4224 * @remarks No-long-jump zone!!!
4225 */
4226static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4227{
4228 AssertPtr(pVCpu);
4229 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4230
4231 /*
4232 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4233 */
4234 int rc = VINF_SUCCESS;
4235 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4236 {
4237#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4238 PVM pVM = pVCpu->CTX_SUFF(pVM);
4239 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4240 uint32_t cGuestMsrs = 0;
4241
4242 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4243 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4244 * when the guest really is in 64-bit mode. */
4245 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4246 if (fSupportsLongMode)
4247 {
4248 pGuestMsr->u32Msr = MSR_K8_LSTAR;
4249 pGuestMsr->u32Reserved = 0;
4250 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4251 pGuestMsr++; cGuestMsrs++;
4252 pGuestMsr->u32Msr = MSR_K6_STAR;
4253 pGuestMsr->u32Reserved = 0;
4254 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4255 pGuestMsr++; cGuestMsrs++;
4256 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4257 pGuestMsr->u32Reserved = 0;
4258 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4259 pGuestMsr++; cGuestMsrs++;
4260 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4261 pGuestMsr->u32Reserved = 0;
4262 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4263 pGuestMsr++; cGuestMsrs++;
4264 }
4265
4266 /*
4267 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4268 * load the guest's copy always (since the MSR bitmap allows passthru unconditionally).
4269 */
4270 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
4271 {
4272 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4273 pGuestMsr->u32Reserved = 0;
4274 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4275 AssertRCReturn(rc, rc);
4276 pGuestMsr++; cGuestMsrs++;
4277 }
4278
4279 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4280 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4281 {
4282 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4283 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4284 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4285 }
4286
4287 /* Update the VCPU's copy of the guest MSR count. */
4288 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4289 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4290 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4291#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4292
4293 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4294 }
4295
4296 /*
4297 * Guest Sysenter MSRs.
4298 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4299 * VM-exits on WRMSRs for these MSRs.
4300 */
4301 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4302 {
4303 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4304 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4305 }
4306
4307 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4308 {
4309 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4310 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4311 }
4312
4313 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4314 {
4315 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4316 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4317 }
4318
4319 return rc;
4320}
4321
4322
4323/**
4324 * Loads the guest activity state into the guest-state area in the VMCS.
4325 *
4326 * @returns VBox status code.
4327 * @param pVCpu Pointer to the VMCPU.
4328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4329 * out-of-sync. Make sure to update the required fields
4330 * before using them.
4331 *
4332 * @remarks No-long-jump zone!!!
4333 */
4334static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4335{
4336 /** @todo See if we can make use of other states, e.g.
4337 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4338 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4339 {
4340 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4341 AssertRCReturn(rc, rc);
4342
4343 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4344 }
4345 return VINF_SUCCESS;
4346}
4347
4348
4349/**
4350 * Sets up the appropriate function to run guest code.
4351 *
4352 * @returns VBox status code.
4353 * @param pVCpu Pointer to the VMCPU.
4354 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4355 * out-of-sync. Make sure to update the required fields
4356 * before using them.
4357 *
4358 * @remarks No-long-jump zone!!!
4359 */
4360static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4361{
4362 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4363 {
4364#ifndef VBOX_ENABLE_64_BITS_GUESTS
4365 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4366#endif
4367 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4368#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4369 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4370 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4371 {
4372 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4373 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4374 }
4375#else
4376 /* 64-bit host or hybrid host. */
4377 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4378#endif
4379 }
4380 else
4381 {
4382 /* Guest is not in long mode, use the 32-bit handler. */
4383#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4384 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4385 {
4386 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4387 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4388 }
4389#else
4390 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4391#endif
4392 }
4393 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4394 return VINF_SUCCESS;
4395}
4396
4397
4398/**
4399 * Wrapper for running the guest code in VT-x.
4400 *
4401 * @returns VBox strict status code.
4402 * @param pVM Pointer to the VM.
4403 * @param pVCpu Pointer to the VMCPU.
4404 * @param pCtx Pointer to the guest-CPU context.
4405 *
4406 * @remarks No-long-jump zone!!!
4407 */
4408DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4409{
4410 /*
4411 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4412 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4413 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4414 */
4415 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4416 /** @todo Add stats for resume vs launch. */
4417#ifdef VBOX_WITH_KERNEL_USING_XMM
4418 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4419#else
4420 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4421#endif
4422}
4423
4424
4425/**
4426 * Reports world-switch error and dumps some useful debug info.
4427 *
4428 * @param pVM Pointer to the VM.
4429 * @param pVCpu Pointer to the VMCPU.
4430 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4431 * @param pCtx Pointer to the guest-CPU context.
4432 * @param pVmxTransient Pointer to the VMX transient structure (only
4433 * exitReason updated).
4434 */
4435static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4436{
4437 Assert(pVM);
4438 Assert(pVCpu);
4439 Assert(pCtx);
4440 Assert(pVmxTransient);
4441 HMVMX_ASSERT_PREEMPT_SAFE();
4442
4443 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4444 switch (rcVMRun)
4445 {
4446 case VERR_VMX_INVALID_VMXON_PTR:
4447 AssertFailed();
4448 break;
4449 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4450 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4451 {
4452 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4453 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4454 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4455 AssertRC(rc);
4456
4457 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4458 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4459 Cannot do it here as we may have been long preempted. */
4460
4461#ifdef VBOX_STRICT
4462 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4463 pVmxTransient->uExitReason));
4464 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4465 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4466 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4467 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4468 else
4469 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4470 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4471 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4472
4473 /* VMX control bits. */
4474 uint32_t u32Val;
4475 uint64_t u64Val;
4476 HMVMXHCUINTREG uHCReg;
4477 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4478 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4479 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4480 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4481 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4482 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4483 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4484 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4485 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4486 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4487 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4488 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4489 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4490 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4491 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4492 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4493 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4494 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4495 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4496 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4497 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4498 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4499 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4500 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4501 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4502 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4503 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4504 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4505 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4506 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4507 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4508 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4509 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4510 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4511 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4512 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4513 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4514 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4515 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4516 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4517 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4518 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4519
4520 /* Guest bits. */
4521 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4522 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4523 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4524 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4525 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4526 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4527 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4528 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4529
4530 /* Host bits. */
4531 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4532 Log4(("Host CR0 %#RHr\n", uHCReg));
4533 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4534 Log4(("Host CR3 %#RHr\n", uHCReg));
4535 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4536 Log4(("Host CR4 %#RHr\n", uHCReg));
4537
4538 RTGDTR HostGdtr;
4539 PCX86DESCHC pDesc;
4540 ASMGetGDTR(&HostGdtr);
4541 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4542 Log4(("Host CS %#08x\n", u32Val));
4543 if (u32Val < HostGdtr.cbGdt)
4544 {
4545 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4546 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4547 }
4548
4549 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4550 Log4(("Host DS %#08x\n", u32Val));
4551 if (u32Val < HostGdtr.cbGdt)
4552 {
4553 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4554 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4555 }
4556
4557 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4558 Log4(("Host ES %#08x\n", u32Val));
4559 if (u32Val < HostGdtr.cbGdt)
4560 {
4561 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4562 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4563 }
4564
4565 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4566 Log4(("Host FS %#08x\n", u32Val));
4567 if (u32Val < HostGdtr.cbGdt)
4568 {
4569 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4570 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4571 }
4572
4573 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4574 Log4(("Host GS %#08x\n", u32Val));
4575 if (u32Val < HostGdtr.cbGdt)
4576 {
4577 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4578 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4579 }
4580
4581 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4582 Log4(("Host SS %#08x\n", u32Val));
4583 if (u32Val < HostGdtr.cbGdt)
4584 {
4585 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4586 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4587 }
4588
4589 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4590 Log4(("Host TR %#08x\n", u32Val));
4591 if (u32Val < HostGdtr.cbGdt)
4592 {
4593 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4594 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4595 }
4596
4597 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4598 Log4(("Host TR Base %#RHv\n", uHCReg));
4599 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4600 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4601 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4602 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4603 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4604 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4605 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4606 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4607 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4608 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4609 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4610 Log4(("Host RSP %#RHv\n", uHCReg));
4611 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4612 Log4(("Host RIP %#RHv\n", uHCReg));
4613# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4614 if (HMVMX_IS_64BIT_HOST_MODE())
4615 {
4616 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4617 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4618 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4619 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4620 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4621 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4622 }
4623# endif
4624#endif /* VBOX_STRICT */
4625 break;
4626 }
4627
4628 default:
4629 /* Impossible */
4630 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4631 break;
4632 }
4633 NOREF(pVM);
4634}
4635
4636
4637#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4638#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4639# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4640#endif
4641#ifdef VBOX_STRICT
4642static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4643{
4644 switch (idxField)
4645 {
4646 case VMX_VMCS_GUEST_RIP:
4647 case VMX_VMCS_GUEST_RSP:
4648 case VMX_VMCS_GUEST_SYSENTER_EIP:
4649 case VMX_VMCS_GUEST_SYSENTER_ESP:
4650 case VMX_VMCS_GUEST_GDTR_BASE:
4651 case VMX_VMCS_GUEST_IDTR_BASE:
4652 case VMX_VMCS_GUEST_CS_BASE:
4653 case VMX_VMCS_GUEST_DS_BASE:
4654 case VMX_VMCS_GUEST_ES_BASE:
4655 case VMX_VMCS_GUEST_FS_BASE:
4656 case VMX_VMCS_GUEST_GS_BASE:
4657 case VMX_VMCS_GUEST_SS_BASE:
4658 case VMX_VMCS_GUEST_LDTR_BASE:
4659 case VMX_VMCS_GUEST_TR_BASE:
4660 case VMX_VMCS_GUEST_CR3:
4661 return true;
4662 }
4663 return false;
4664}
4665
4666static bool hmR0VmxIsValidReadField(uint32_t idxField)
4667{
4668 switch (idxField)
4669 {
4670 /* Read-only fields. */
4671 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4672 return true;
4673 }
4674 /* Remaining readable fields should also be writable. */
4675 return hmR0VmxIsValidWriteField(idxField);
4676}
4677#endif /* VBOX_STRICT */
4678
4679
4680/**
4681 * Executes the specified handler in 64-bit mode.
4682 *
4683 * @returns VBox status code.
4684 * @param pVM Pointer to the VM.
4685 * @param pVCpu Pointer to the VMCPU.
4686 * @param pCtx Pointer to the guest CPU context.
4687 * @param enmOp The operation to perform.
4688 * @param cbParam Number of parameters.
4689 * @param paParam Array of 32-bit parameters.
4690 */
4691VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4692 uint32_t *paParam)
4693{
4694 int rc, rc2;
4695 PHMGLOBALCPUINFO pCpu;
4696 RTHCPHYS HCPhysCpuPage;
4697 RTCCUINTREG uOldEflags;
4698
4699 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4700 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4701 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4702 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4703
4704#ifdef VBOX_STRICT
4705 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4706 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4707
4708 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4709 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4710#endif
4711
4712 /* Disable interrupts. */
4713 uOldEflags = ASMIntDisableFlags();
4714
4715#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4716 RTCPUID idHostCpu = RTMpCpuId();
4717 CPUMR0SetLApic(pVCpu, idHostCpu);
4718#endif
4719
4720 pCpu = HMR0GetCurrentCpu();
4721 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4722
4723 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4724 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4725
4726 /* Leave VMX Root Mode. */
4727 VMXDisable();
4728
4729 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4730
4731 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4732 CPUMSetHyperEIP(pVCpu, enmOp);
4733 for (int i = (int)cbParam - 1; i >= 0; i--)
4734 CPUMPushHyper(pVCpu, paParam[i]);
4735
4736 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4737
4738 /* Call the switcher. */
4739 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4740 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4741
4742 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4743 /* Make sure the VMX instructions don't cause #UD faults. */
4744 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4745
4746 /* Re-enter VMX Root Mode */
4747 rc2 = VMXEnable(HCPhysCpuPage);
4748 if (RT_FAILURE(rc2))
4749 {
4750 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4751 ASMSetFlags(uOldEflags);
4752 return rc2;
4753 }
4754
4755 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4756 AssertRC(rc2);
4757 Assert(!(ASMGetFlags() & X86_EFL_IF));
4758 ASMSetFlags(uOldEflags);
4759 return rc;
4760}
4761
4762
4763/**
4764 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4765 * supporting 64-bit guests.
4766 *
4767 * @returns VBox status code.
4768 * @param fResume Whether to VMLAUNCH or VMRESUME.
4769 * @param pCtx Pointer to the guest-CPU context.
4770 * @param pCache Pointer to the VMCS cache.
4771 * @param pVM Pointer to the VM.
4772 * @param pVCpu Pointer to the VMCPU.
4773 */
4774DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4775{
4776 uint32_t aParam[6];
4777 PHMGLOBALCPUINFO pCpu = NULL;
4778 RTHCPHYS HCPhysCpuPage = 0;
4779 int rc = VERR_INTERNAL_ERROR_5;
4780
4781 pCpu = HMR0GetCurrentCpu();
4782 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4783
4784#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4785 pCache->uPos = 1;
4786 pCache->interPD = PGMGetInterPaeCR3(pVM);
4787 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4788#endif
4789
4790#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4791 pCache->TestIn.HCPhysCpuPage = 0;
4792 pCache->TestIn.HCPhysVmcs = 0;
4793 pCache->TestIn.pCache = 0;
4794 pCache->TestOut.HCPhysVmcs = 0;
4795 pCache->TestOut.pCache = 0;
4796 pCache->TestOut.pCtx = 0;
4797 pCache->TestOut.eflags = 0;
4798#endif
4799
4800 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4801 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4802 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4803 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4804 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4805 aParam[5] = 0;
4806
4807#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4808 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4809 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4810#endif
4811 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4812
4813#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4814 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4815 Assert(pCtx->dr[4] == 10);
4816 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4817#endif
4818
4819#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4820 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4821 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4822 pVCpu->hm.s.vmx.HCPhysVmcs));
4823 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4824 pCache->TestOut.HCPhysVmcs));
4825 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4826 pCache->TestOut.pCache));
4827 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4828 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4829 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4830 pCache->TestOut.pCtx));
4831 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4832#endif
4833 return rc;
4834}
4835
4836
4837/**
4838 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4839 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4840 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4841 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4842 *
4843 * @returns VBox status code.
4844 * @param pVM Pointer to the VM.
4845 * @param pVCpu Pointer to the VMCPU.
4846 */
4847static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4848{
4849#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4850{ \
4851 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4852 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4853 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4854 ++cReadFields; \
4855}
4856
4857 AssertPtr(pVM);
4858 AssertPtr(pVCpu);
4859 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4860 uint32_t cReadFields = 0;
4861
4862 /*
4863 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4864 * and serve to indicate exceptions to the rules.
4865 */
4866
4867 /* Guest-natural selector base fields. */
4868#if 0
4869 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4870 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4871 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4872#endif
4873 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4874 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4875 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4876 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4877 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4878 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4879 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4880 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4881 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4882 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4883 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4884 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4885#if 0
4886 /* Unused natural width guest-state fields. */
4887 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4888 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4889#endif
4890 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4891 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4892
4893 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4894#if 0
4895 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4896 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4897 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4898 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4899 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4900 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4901 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4902 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4903 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4904#endif
4905
4906 /* Natural width guest-state fields. */
4907 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4908#if 0
4909 /* Currently unused field. */
4910 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4911#endif
4912
4913 if (pVM->hm.s.fNestedPaging)
4914 {
4915 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4916 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4917 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4918 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4919 }
4920 else
4921 {
4922 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4923 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4924 }
4925
4926#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4927 return VINF_SUCCESS;
4928}
4929
4930
4931/**
4932 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4933 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4934 * darwin, running 64-bit guests).
4935 *
4936 * @returns VBox status code.
4937 * @param pVCpu Pointer to the VMCPU.
4938 * @param idxField The VMCS field encoding.
4939 * @param u64Val 16, 32 or 64 bits value.
4940 */
4941VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4942{
4943 int rc;
4944 switch (idxField)
4945 {
4946 /*
4947 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4948 */
4949 /* 64-bit Control fields. */
4950 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4951 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4952 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4953 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4954 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4955 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4956 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4957 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4958 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4959 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4960 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4961 case VMX_VMCS64_CTRL_EPTP_FULL:
4962 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4963 /* 64-bit Guest-state fields. */
4964 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4965 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4966 case VMX_VMCS64_GUEST_PAT_FULL:
4967 case VMX_VMCS64_GUEST_EFER_FULL:
4968 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4969 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4970 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4971 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4972 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4973 /* 64-bit Host-state fields. */
4974 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4975 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4976 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4977 {
4978 rc = VMXWriteVmcs32(idxField, u64Val);
4979 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4980 break;
4981 }
4982
4983 /*
4984 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4985 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4986 */
4987 /* Natural-width Guest-state fields. */
4988 case VMX_VMCS_GUEST_CR3:
4989 case VMX_VMCS_GUEST_ES_BASE:
4990 case VMX_VMCS_GUEST_CS_BASE:
4991 case VMX_VMCS_GUEST_SS_BASE:
4992 case VMX_VMCS_GUEST_DS_BASE:
4993 case VMX_VMCS_GUEST_FS_BASE:
4994 case VMX_VMCS_GUEST_GS_BASE:
4995 case VMX_VMCS_GUEST_LDTR_BASE:
4996 case VMX_VMCS_GUEST_TR_BASE:
4997 case VMX_VMCS_GUEST_GDTR_BASE:
4998 case VMX_VMCS_GUEST_IDTR_BASE:
4999 case VMX_VMCS_GUEST_RSP:
5000 case VMX_VMCS_GUEST_RIP:
5001 case VMX_VMCS_GUEST_SYSENTER_ESP:
5002 case VMX_VMCS_GUEST_SYSENTER_EIP:
5003 {
5004 if (!(u64Val >> 32))
5005 {
5006 /* If this field is 64-bit, VT-x will zero out the top bits. */
5007 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5008 }
5009 else
5010 {
5011 /* Assert that only the 32->64 switcher case should ever come here. */
5012 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5013 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5014 }
5015 break;
5016 }
5017
5018 default:
5019 {
5020 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5021 rc = VERR_INVALID_PARAMETER;
5022 break;
5023 }
5024 }
5025 AssertRCReturn(rc, rc);
5026 return rc;
5027}
5028
5029
5030/**
5031 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5032 * hosts (except darwin) for 64-bit guests.
5033 *
5034 * @param pVCpu Pointer to the VMCPU.
5035 * @param idxField The VMCS field encoding.
5036 * @param u64Val 16, 32 or 64 bits value.
5037 */
5038VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5039{
5040 AssertPtr(pVCpu);
5041 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5042
5043 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5044 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5045
5046 /* Make sure there are no duplicates. */
5047 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5048 {
5049 if (pCache->Write.aField[i] == idxField)
5050 {
5051 pCache->Write.aFieldVal[i] = u64Val;
5052 return VINF_SUCCESS;
5053 }
5054 }
5055
5056 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5057 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5058 pCache->Write.cValidEntries++;
5059 return VINF_SUCCESS;
5060}
5061
5062/* Enable later when the assembly code uses these as callbacks. */
5063#if 0
5064/*
5065 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5066 *
5067 * @param pVCpu Pointer to the VMCPU.
5068 * @param pCache Pointer to the VMCS cache.
5069 *
5070 * @remarks No-long-jump zone!!!
5071 */
5072VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5073{
5074 AssertPtr(pCache);
5075 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5076 {
5077 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5078 AssertRC(rc);
5079 }
5080 pCache->Write.cValidEntries = 0;
5081}
5082
5083
5084/**
5085 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5086 *
5087 * @param pVCpu Pointer to the VMCPU.
5088 * @param pCache Pointer to the VMCS cache.
5089 *
5090 * @remarks No-long-jump zone!!!
5091 */
5092VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5093{
5094 AssertPtr(pCache);
5095 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5096 {
5097 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5098 AssertRC(rc);
5099 }
5100}
5101#endif
5102#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5103
5104
5105/**
5106 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5107 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5108 * timer.
5109 *
5110 * @returns VBox status code.
5111 * @param pVCpu Pointer to the VMCPU.
5112 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5113 * out-of-sync. Make sure to update the required fields
5114 * before using them.
5115 * @remarks No-long-jump zone!!!
5116 */
5117static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5118{
5119 int rc = VERR_INTERNAL_ERROR_5;
5120 bool fOffsettedTsc = false;
5121 PVM pVM = pVCpu->CTX_SUFF(pVM);
5122 if (pVM->hm.s.vmx.fUsePreemptTimer)
5123 {
5124 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5125
5126 /* Make sure the returned values have sane upper and lower boundaries. */
5127 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5128 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5129 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5130 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5131
5132 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5133 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5134 }
5135 else
5136 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5137
5138 if (fOffsettedTsc)
5139 {
5140 uint64_t u64CurTSC = ASMReadTSC();
5141 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5142 {
5143 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5144 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5145
5146 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5147 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5148 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5149 }
5150 else
5151 {
5152 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5153 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5154 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5155 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5156 }
5157 }
5158 else
5159 {
5160 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5161 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5162 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5163 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5164 }
5165}
5166
5167
5168/**
5169 * Determines if an exception is a contributory exception. Contributory
5170 * exceptions are ones which can cause double-faults. Page-fault is
5171 * intentionally not included here as it's a conditional contributory exception.
5172 *
5173 * @returns true if the exception is contributory, false otherwise.
5174 * @param uVector The exception vector.
5175 */
5176DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5177{
5178 switch (uVector)
5179 {
5180 case X86_XCPT_GP:
5181 case X86_XCPT_SS:
5182 case X86_XCPT_NP:
5183 case X86_XCPT_TS:
5184 case X86_XCPT_DE:
5185 return true;
5186 default:
5187 break;
5188 }
5189 return false;
5190}
5191
5192
5193/**
5194 * Sets an event as a pending event to be injected into the guest.
5195 *
5196 * @param pVCpu Pointer to the VMCPU.
5197 * @param u32IntInfo The VM-entry interruption-information field.
5198 * @param cbInstr The VM-entry instruction length in bytes (for software
5199 * interrupts, exceptions and privileged software
5200 * exceptions).
5201 * @param u32ErrCode The VM-entry exception error code.
5202 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5203 * page-fault.
5204 *
5205 * @remarks Statistics counter assumes this is a guest event being injected or
5206 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5207 * always incremented.
5208 */
5209DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5210 RTGCUINTPTR GCPtrFaultAddress)
5211{
5212 Assert(!pVCpu->hm.s.Event.fPending);
5213 pVCpu->hm.s.Event.fPending = true;
5214 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5215 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5216 pVCpu->hm.s.Event.cbInstr = cbInstr;
5217 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5218
5219 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5220}
5221
5222
5223/**
5224 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5225 *
5226 * @param pVCpu Pointer to the VMCPU.
5227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5228 * out-of-sync. Make sure to update the required fields
5229 * before using them.
5230 */
5231DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5232{
5233 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5234 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5235 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5236 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5237}
5238
5239
5240/**
5241 * Handle a condition that occurred while delivering an event through the guest
5242 * IDT.
5243 *
5244 * @returns VBox status code (informational error codes included).
5245 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5246 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5247 * continue execution of the guest which will delivery the #DF.
5248 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5249 *
5250 * @param pVCpu Pointer to the VMCPU.
5251 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5252 * out-of-sync. Make sure to update the required fields
5253 * before using them.
5254 * @param pVmxTransient Pointer to the VMX transient structure.
5255 *
5256 * @remarks No-long-jump zone!!!
5257 */
5258static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5259{
5260 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5261 AssertRCReturn(rc, rc);
5262 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5263 {
5264 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5265 AssertRCReturn(rc, rc);
5266
5267 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5268 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5269 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5270
5271 typedef enum
5272 {
5273 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5274 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5275 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5276 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5277 } VMXREFLECTXCPT;
5278
5279 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5280 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5281 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5282 {
5283 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5284 {
5285 enmReflect = VMXREFLECTXCPT_XCPT;
5286#ifdef VBOX_STRICT
5287 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5288 && uExitVector == X86_XCPT_PF)
5289 {
5290 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5291 }
5292#endif
5293 if ( uExitVector == X86_XCPT_PF
5294 && uIdtVector == X86_XCPT_PF)
5295 {
5296 pVmxTransient->fVectoringPF = true;
5297 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5298 }
5299 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5300 && hmR0VmxIsContributoryXcpt(uExitVector)
5301 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5302 || uIdtVector == X86_XCPT_PF))
5303 {
5304 enmReflect = VMXREFLECTXCPT_DF;
5305 }
5306 else if (uIdtVector == X86_XCPT_DF)
5307 enmReflect = VMXREFLECTXCPT_TF;
5308 }
5309 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5310 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5311 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5312 {
5313 /*
5314 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5315 * (whatever they are) as they reoccur when restarting the instruction.
5316 */
5317 enmReflect = VMXREFLECTXCPT_XCPT;
5318 }
5319 }
5320 else
5321 {
5322 /*
5323 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5324 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5325 * original exception to the guest after handling the VM-exit.
5326 */
5327 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5328 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5329 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5330 {
5331 enmReflect = VMXREFLECTXCPT_XCPT;
5332 }
5333 }
5334
5335 switch (enmReflect)
5336 {
5337 case VMXREFLECTXCPT_XCPT:
5338 {
5339 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5340 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5341 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5342
5343 uint32_t u32ErrCode = 0;
5344 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5345 {
5346 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5347 AssertRCReturn(rc, rc);
5348 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5349 }
5350
5351 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5352 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5353 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5354 rc = VINF_SUCCESS;
5355 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5356 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5357
5358 break;
5359 }
5360
5361 case VMXREFLECTXCPT_DF:
5362 {
5363 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5364 rc = VINF_HM_DOUBLE_FAULT;
5365 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5366 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5367
5368 break;
5369 }
5370
5371 case VMXREFLECTXCPT_TF:
5372 {
5373 rc = VINF_EM_RESET;
5374 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5375 uExitVector));
5376 break;
5377 }
5378
5379 default:
5380 Assert(rc == VINF_SUCCESS);
5381 break;
5382 }
5383 }
5384 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5385 return rc;
5386}
5387
5388
5389/**
5390 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5391 *
5392 * @returns VBox status code.
5393 * @param pVCpu Pointer to the VMCPU.
5394 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5395 * out-of-sync. Make sure to update the required fields
5396 * before using them.
5397 *
5398 * @remarks No-long-jump zone!!!
5399 */
5400static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5401{
5402 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5403 {
5404 uint32_t uVal = 0;
5405 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5406 AssertRCReturn(rc, rc);
5407
5408 uint32_t uShadow = 0;
5409 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5410 AssertRCReturn(rc, rc);
5411
5412 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5413 CPUMSetGuestCR0(pVCpu, uVal);
5414 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5415 }
5416 return VINF_SUCCESS;
5417}
5418
5419
5420/**
5421 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5422 *
5423 * @returns VBox status code.
5424 * @param pVCpu Pointer to the VMCPU.
5425 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5426 * out-of-sync. Make sure to update the required fields
5427 * before using them.
5428 *
5429 * @remarks No-long-jump zone!!!
5430 */
5431static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5432{
5433 int rc = VINF_SUCCESS;
5434 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5435 {
5436 uint32_t uVal = 0;
5437 uint32_t uShadow = 0;
5438 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5439 AssertRCReturn(rc, rc);
5440 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5441 AssertRCReturn(rc, rc);
5442
5443 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5444 CPUMSetGuestCR4(pVCpu, uVal);
5445 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5446 }
5447 return rc;
5448}
5449
5450
5451/**
5452 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5453 *
5454 * @returns VBox status code.
5455 * @param pVCpu Pointer to the VMCPU.
5456 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5457 * out-of-sync. Make sure to update the required fields
5458 * before using them.
5459 *
5460 * @remarks No-long-jump zone!!!
5461 */
5462static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5463{
5464 int rc = VINF_SUCCESS;
5465 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5466 {
5467 uint64_t u64Val = 0;
5468 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5469 AssertRCReturn(rc, rc);
5470
5471 pMixedCtx->rip = u64Val;
5472 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5473 }
5474 return rc;
5475}
5476
5477
5478/**
5479 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5480 *
5481 * @returns VBox status code.
5482 * @param pVCpu Pointer to the VMCPU.
5483 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5484 * out-of-sync. Make sure to update the required fields
5485 * before using them.
5486 *
5487 * @remarks No-long-jump zone!!!
5488 */
5489static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5490{
5491 int rc = VINF_SUCCESS;
5492 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5493 {
5494 uint64_t u64Val = 0;
5495 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5496 AssertRCReturn(rc, rc);
5497
5498 pMixedCtx->rsp = u64Val;
5499 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5500 }
5501 return rc;
5502}
5503
5504
5505/**
5506 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5507 *
5508 * @returns VBox status code.
5509 * @param pVCpu Pointer to the VMCPU.
5510 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5511 * out-of-sync. Make sure to update the required fields
5512 * before using them.
5513 *
5514 * @remarks No-long-jump zone!!!
5515 */
5516static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5517{
5518 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5519 {
5520 uint32_t uVal = 0;
5521 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5522 AssertRCReturn(rc, rc);
5523
5524 pMixedCtx->eflags.u32 = uVal;
5525 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5526 {
5527 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5528 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5529
5530 pMixedCtx->eflags.Bits.u1VM = 0;
5531 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5532 }
5533
5534 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5535 }
5536 return VINF_SUCCESS;
5537}
5538
5539
5540/**
5541 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5542 * guest-CPU context.
5543 */
5544DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5545{
5546 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5547 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5548 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5549 return rc;
5550}
5551
5552
5553/**
5554 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5555 * from the guest-state area in the VMCS.
5556 *
5557 * @param pVCpu Pointer to the VMCPU.
5558 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5559 * out-of-sync. Make sure to update the required fields
5560 * before using them.
5561 *
5562 * @remarks No-long-jump zone!!!
5563 */
5564static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5565{
5566 uint32_t uIntrState = 0;
5567 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5568 AssertRC(rc);
5569
5570 if (!uIntrState)
5571 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5572 else
5573 {
5574 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5575 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5576 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5577 AssertRC(rc);
5578 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5579 AssertRC(rc);
5580
5581 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5582 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5583 }
5584}
5585
5586
5587/**
5588 * Saves the guest's activity state.
5589 *
5590 * @returns VBox status code.
5591 * @param pVCpu Pointer to the VMCPU.
5592 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5593 * out-of-sync. Make sure to update the required fields
5594 * before using them.
5595 *
5596 * @remarks No-long-jump zone!!!
5597 */
5598static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5599{
5600 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5601 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5602 return VINF_SUCCESS;
5603}
5604
5605
5606/**
5607 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5608 * the current VMCS into the guest-CPU context.
5609 *
5610 * @returns VBox status code.
5611 * @param pVCpu Pointer to the VMCPU.
5612 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5613 * out-of-sync. Make sure to update the required fields
5614 * before using them.
5615 *
5616 * @remarks No-long-jump zone!!!
5617 */
5618static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5619{
5620 int rc = VINF_SUCCESS;
5621 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5622 {
5623 uint32_t u32Val = 0;
5624 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5625 pMixedCtx->SysEnter.cs = u32Val;
5626 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5627 }
5628
5629 uint64_t u64Val = 0;
5630 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5631 {
5632 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5633 pMixedCtx->SysEnter.eip = u64Val;
5634 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5635 }
5636 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5637 {
5638 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5639 pMixedCtx->SysEnter.esp = u64Val;
5640 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5641 }
5642 return rc;
5643}
5644
5645
5646/**
5647 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5648 * context.
5649 *
5650 * @returns VBox status code.
5651 * @param pVCpu Pointer to the VMCPU.
5652 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5653 * out-of-sync. Make sure to update the required fields
5654 * before using them.
5655 *
5656 * @remarks No-long-jump zone!!!
5657 */
5658static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5659{
5660 int rc = VINF_SUCCESS;
5661 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5662 {
5663 uint64_t u64Val = 0;
5664 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5665 pMixedCtx->fs.u64Base = u64Val;
5666 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5667 }
5668 return rc;
5669}
5670
5671
5672/**
5673 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5674 * context.
5675 *
5676 * @returns VBox status code.
5677 * @param pVCpu Pointer to the VMCPU.
5678 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5679 * out-of-sync. Make sure to update the required fields
5680 * before using them.
5681 *
5682 * @remarks No-long-jump zone!!!
5683 */
5684static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5685{
5686 int rc = VINF_SUCCESS;
5687 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5688 {
5689 uint64_t u64Val = 0;
5690 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5691 pMixedCtx->gs.u64Base = u64Val;
5692 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5693 }
5694 return rc;
5695}
5696
5697
5698/**
5699 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5700 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5701 * and TSC_AUX.
5702 *
5703 * @returns VBox status code.
5704 * @param pVCpu Pointer to the VMCPU.
5705 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5706 * out-of-sync. Make sure to update the required fields
5707 * before using them.
5708 *
5709 * @remarks No-long-jump zone!!!
5710 */
5711static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5712{
5713 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5714 return VINF_SUCCESS;
5715
5716#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5717 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5718 {
5719 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5720 pMsr += i;
5721 switch (pMsr->u32Msr)
5722 {
5723 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5724 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5725 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5726 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5727 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5728 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5729 default:
5730 {
5731 AssertFailed();
5732 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5733 }
5734 }
5735 }
5736#endif
5737
5738 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5739 return VINF_SUCCESS;
5740}
5741
5742
5743/**
5744 * Saves the guest control registers from the current VMCS into the guest-CPU
5745 * context.
5746 *
5747 * @returns VBox status code.
5748 * @param pVCpu Pointer to the VMCPU.
5749 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5750 * out-of-sync. Make sure to update the required fields
5751 * before using them.
5752 *
5753 * @remarks No-long-jump zone!!!
5754 */
5755static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5756{
5757 /* Guest CR0. Guest FPU. */
5758 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5759 AssertRCReturn(rc, rc);
5760
5761 /* Guest CR4. */
5762 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5763 AssertRCReturn(rc, rc);
5764
5765 /* Guest CR2 - updated always during the world-switch or in #PF. */
5766 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5767 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5768 {
5769 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5770 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5771
5772 PVM pVM = pVCpu->CTX_SUFF(pVM);
5773 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5774 || ( pVM->hm.s.fNestedPaging
5775 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5776 {
5777 uint64_t u64Val = 0;
5778 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5779 if (pMixedCtx->cr3 != u64Val)
5780 {
5781 CPUMSetGuestCR3(pVCpu, u64Val);
5782 if (VMMRZCallRing3IsEnabled(pVCpu))
5783 {
5784 PGMUpdateCR3(pVCpu, u64Val);
5785 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5786 }
5787 else
5788 {
5789 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5790 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5791 }
5792 }
5793
5794 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5795 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5796 {
5797 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5798 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5799 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5800 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5801
5802 if (VMMRZCallRing3IsEnabled(pVCpu))
5803 {
5804 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5805 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5806 }
5807 else
5808 {
5809 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5810 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5811 }
5812 }
5813 }
5814
5815 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5816 }
5817
5818 /*
5819 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5820 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5821 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5822 *
5823 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5824 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5825 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5826 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5827 *
5828 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5829 */
5830 if (VMMRZCallRing3IsEnabled(pVCpu))
5831 {
5832 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5833 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5834
5835 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5836 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5837
5838 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5839 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5840 }
5841
5842 return rc;
5843}
5844
5845
5846/**
5847 * Reads a guest segment register from the current VMCS into the guest-CPU
5848 * context.
5849 *
5850 * @returns VBox status code.
5851 * @param pVCpu Pointer to the VMCPU.
5852 * @param idxSel Index of the selector in the VMCS.
5853 * @param idxLimit Index of the segment limit in the VMCS.
5854 * @param idxBase Index of the segment base in the VMCS.
5855 * @param idxAccess Index of the access rights of the segment in the VMCS.
5856 * @param pSelReg Pointer to the segment selector.
5857 *
5858 * @remarks No-long-jump zone!!!
5859 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5860 * macro as that takes care of whether to read from the VMCS cache or
5861 * not.
5862 */
5863DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5864 PCPUMSELREG pSelReg)
5865{
5866 uint32_t u32Val = 0;
5867 int rc = VMXReadVmcs32(idxSel, &u32Val);
5868 AssertRCReturn(rc, rc);
5869 pSelReg->Sel = (uint16_t)u32Val;
5870 pSelReg->ValidSel = (uint16_t)u32Val;
5871 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5872
5873 rc = VMXReadVmcs32(idxLimit, &u32Val);
5874 AssertRCReturn(rc, rc);
5875 pSelReg->u32Limit = u32Val;
5876
5877 uint64_t u64Val = 0;
5878 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5879 AssertRCReturn(rc, rc);
5880 pSelReg->u64Base = u64Val;
5881
5882 rc = VMXReadVmcs32(idxAccess, &u32Val);
5883 AssertRCReturn(rc, rc);
5884 pSelReg->Attr.u = u32Val;
5885
5886 /*
5887 * If VT-x marks the segment as unusable, most other bits remain undefined:
5888 * - For CS the L, D and G bits have meaning.
5889 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5890 * - For the remaining data segments no bits are defined.
5891 *
5892 * The present bit and the unusable bit has been observed to be set at the
5893 * same time (the selector was supposed to invalid as we started executing
5894 * a V8086 interrupt in ring-0).
5895 *
5896 * What should be important for the rest of the VBox code that the P bit is
5897 * cleared. Some of the other VBox code recognizes the unusable bit, but
5898 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5899 * safe side here, we'll strip off P and other bits we don't care about. If
5900 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5901 *
5902 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5903 */
5904 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5905 {
5906 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5907
5908 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5909 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5910 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5911
5912 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5913#ifdef DEBUG_bird
5914 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5915 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5916 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5917#endif
5918 }
5919 return VINF_SUCCESS;
5920}
5921
5922
5923#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5924# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5925 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5926 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5927#else
5928# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5929 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5930 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5931#endif
5932
5933
5934/**
5935 * Saves the guest segment registers from the current VMCS into the guest-CPU
5936 * context.
5937 *
5938 * @returns VBox status code.
5939 * @param pVCpu Pointer to the VMCPU.
5940 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5941 * out-of-sync. Make sure to update the required fields
5942 * before using them.
5943 *
5944 * @remarks No-long-jump zone!!!
5945 */
5946static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5947{
5948 /* Guest segment registers. */
5949 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5950 {
5951 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5952 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5953 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5954 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5955 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5956 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5957 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5958
5959 /* Restore segment attributes for real-on-v86 mode hack. */
5960 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5961 {
5962 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5963 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5964 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5965 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5966 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5967 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5968 }
5969 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5970 }
5971
5972 return VINF_SUCCESS;
5973}
5974
5975
5976/**
5977 * Saves the guest descriptor table registers and task register from the current
5978 * VMCS into the guest-CPU context.
5979 *
5980 * @returns VBox status code.
5981 * @param pVCpu Pointer to the VMCPU.
5982 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5983 * out-of-sync. Make sure to update the required fields
5984 * before using them.
5985 *
5986 * @remarks No-long-jump zone!!!
5987 */
5988static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5989{
5990 int rc = VINF_SUCCESS;
5991
5992 /* Guest LDTR. */
5993 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5994 {
5995 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5996 AssertRCReturn(rc, rc);
5997 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5998 }
5999
6000 /* Guest GDTR. */
6001 uint64_t u64Val = 0;
6002 uint32_t u32Val = 0;
6003 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6004 {
6005 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6006 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6007 pMixedCtx->gdtr.pGdt = u64Val;
6008 pMixedCtx->gdtr.cbGdt = u32Val;
6009 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
6010 }
6011
6012 /* Guest IDTR. */
6013 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6014 {
6015 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6016 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6017 pMixedCtx->idtr.pIdt = u64Val;
6018 pMixedCtx->idtr.cbIdt = u32Val;
6019 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
6020 }
6021
6022 /* Guest TR. */
6023 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6024 {
6025 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6026 AssertRCReturn(rc, rc);
6027
6028 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6029 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6030 {
6031 rc = VMXLOCAL_READ_SEG(TR, tr);
6032 AssertRCReturn(rc, rc);
6033 }
6034 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
6035 }
6036 return rc;
6037}
6038
6039#undef VMXLOCAL_READ_SEG
6040
6041
6042/**
6043 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6044 * context.
6045 *
6046 * @returns VBox status code.
6047 * @param pVCpu Pointer to the VMCPU.
6048 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6049 * out-of-sync. Make sure to update the required fields
6050 * before using them.
6051 *
6052 * @remarks No-long-jump zone!!!
6053 */
6054static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6055{
6056 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6057 {
6058 if (!pVCpu->hm.s.fUsingHyperDR7)
6059 {
6060 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6061 uint32_t u32Val;
6062 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6063 pMixedCtx->dr[7] = u32Val;
6064 }
6065
6066 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
6067 }
6068 return VINF_SUCCESS;
6069}
6070
6071
6072/**
6073 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6074 *
6075 * @returns VBox status code.
6076 * @param pVCpu Pointer to the VMCPU.
6077 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6078 * out-of-sync. Make sure to update the required fields
6079 * before using them.
6080 *
6081 * @remarks No-long-jump zone!!!
6082 */
6083static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6084{
6085 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6086 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
6087 return VINF_SUCCESS;
6088}
6089
6090
6091/**
6092 * Saves the entire guest state from the currently active VMCS into the
6093 * guest-CPU context. This essentially VMREADs all guest-data.
6094 *
6095 * @returns VBox status code.
6096 * @param pVCpu Pointer to the VMCPU.
6097 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6098 * out-of-sync. Make sure to update the required fields
6099 * before using them.
6100 */
6101static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6102{
6103 Assert(pVCpu);
6104 Assert(pMixedCtx);
6105
6106 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6107 return VINF_SUCCESS;
6108
6109 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6110 again on the ring-3 callback path, there is no real need to. */
6111 if (VMMRZCallRing3IsEnabled(pVCpu))
6112 VMMR0LogFlushDisable(pVCpu);
6113 else
6114 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6115 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6116
6117 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6118 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6119
6120 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6121 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6122
6123 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6124 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6125
6126 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6127 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6128
6129 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6130 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6131
6132 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6133 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6134
6135 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
6136 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6137
6138 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
6139 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6140
6141 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6142 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6143
6144 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6145 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6146
6147 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6148 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6149
6150 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6151 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6152
6153 if (VMMRZCallRing3IsEnabled(pVCpu))
6154 VMMR0LogFlushEnable(pVCpu);
6155
6156 return rc;
6157}
6158
6159
6160/**
6161 * Check per-VM and per-VCPU force flag actions that require us to go back to
6162 * ring-3 for one reason or another.
6163 *
6164 * @returns VBox status code (information status code included).
6165 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6166 * ring-3.
6167 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6168 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6169 * interrupts)
6170 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6171 * all EMTs to be in ring-3.
6172 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6173 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6174 * to the EM loop.
6175 *
6176 * @param pVM Pointer to the VM.
6177 * @param pVCpu Pointer to the VMCPU.
6178 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6179 * out-of-sync. Make sure to update the required fields
6180 * before using them.
6181 */
6182static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6183{
6184 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6185
6186 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6187 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6188 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6189 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6190 {
6191 /* We need the control registers now, make sure the guest-CPU context is updated. */
6192 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6193 AssertRCReturn(rc3, rc3);
6194
6195 /* Pending HM CR3 sync. */
6196 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6197 {
6198 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6199 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6200 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6201 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6202 }
6203
6204 /* Pending HM PAE PDPEs. */
6205 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6206 {
6207 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6208 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6209 }
6210
6211 /* Pending PGM C3 sync. */
6212 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6213 {
6214 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6215 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6216 if (rc2 != VINF_SUCCESS)
6217 {
6218 AssertRC(rc2);
6219 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6220 return rc2;
6221 }
6222 }
6223
6224 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6225 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6226 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6227 {
6228 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6229 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6230 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6231 return rc2;
6232 }
6233
6234 /* Pending VM request packets, such as hardware interrupts. */
6235 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6236 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6237 {
6238 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6239 return VINF_EM_PENDING_REQUEST;
6240 }
6241
6242 /* Pending PGM pool flushes. */
6243 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6244 {
6245 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6246 return VINF_PGM_POOL_FLUSH_PENDING;
6247 }
6248
6249 /* Pending DMA requests. */
6250 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6251 {
6252 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6253 return VINF_EM_RAW_TO_R3;
6254 }
6255 }
6256
6257 return VINF_SUCCESS;
6258}
6259
6260
6261/**
6262 * Converts any TRPM trap into a pending HM event. This is typically used when
6263 * entering from ring-3 (not longjmp returns).
6264 *
6265 * @param pVCpu Pointer to the VMCPU.
6266 */
6267static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6268{
6269 Assert(TRPMHasTrap(pVCpu));
6270 Assert(!pVCpu->hm.s.Event.fPending);
6271
6272 uint8_t uVector;
6273 TRPMEVENT enmTrpmEvent;
6274 RTGCUINT uErrCode;
6275 RTGCUINTPTR GCPtrFaultAddress;
6276 uint8_t cbInstr;
6277
6278 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6279 AssertRC(rc);
6280
6281 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6282 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6283 if (enmTrpmEvent == TRPM_TRAP)
6284 {
6285 switch (uVector)
6286 {
6287 case X86_XCPT_BP:
6288 case X86_XCPT_OF:
6289 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6290 break;
6291
6292 case X86_XCPT_PF:
6293 case X86_XCPT_DF:
6294 case X86_XCPT_TS:
6295 case X86_XCPT_NP:
6296 case X86_XCPT_SS:
6297 case X86_XCPT_GP:
6298 case X86_XCPT_AC:
6299 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6300 /* no break! */
6301 default:
6302 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6303 break;
6304 }
6305 }
6306 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6307 {
6308 if (uVector == X86_XCPT_NMI)
6309 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6310 else
6311 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6312 }
6313 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6314 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6315 else
6316 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6317
6318 rc = TRPMResetTrap(pVCpu);
6319 AssertRC(rc);
6320 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6321 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6322
6323 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6324 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6325}
6326
6327
6328/**
6329 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6330 * VT-x to execute any instruction.
6331 *
6332 * @param pvCpu Pointer to the VMCPU.
6333 */
6334static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6335{
6336 Assert(pVCpu->hm.s.Event.fPending);
6337
6338 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6339 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6340 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6341 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6342
6343 /* If a trap was already pending, we did something wrong! */
6344 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6345
6346 TRPMEVENT enmTrapType;
6347 switch (uVectorType)
6348 {
6349 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6350 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6351 enmTrapType = TRPM_HARDWARE_INT;
6352 break;
6353
6354 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6355 enmTrapType = TRPM_SOFTWARE_INT;
6356 break;
6357
6358 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6359 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6360 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6361 enmTrapType = TRPM_TRAP;
6362 break;
6363
6364 default:
6365 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6366 enmTrapType = TRPM_32BIT_HACK;
6367 break;
6368 }
6369
6370 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6371
6372 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6373 AssertRC(rc);
6374
6375 if (fErrorCodeValid)
6376 TRPMSetErrorCode(pVCpu, uErrorCode);
6377
6378 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6379 && uVector == X86_XCPT_PF)
6380 {
6381 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6382 }
6383 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6384 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6385 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6386 {
6387 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6388 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6389 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6390 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6391 }
6392 pVCpu->hm.s.Event.fPending = false;
6393}
6394
6395
6396/**
6397 * Does the necessary state syncing before returning to ring-3 for any reason
6398 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6399 *
6400 * @returns VBox status code.
6401 * @param pVM Pointer to the VM.
6402 * @param pVCpu Pointer to the VMCPU.
6403 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6404 * be out-of-sync. Make sure to update the required
6405 * fields before using them.
6406 * @param fSaveGuestState Whether to save the guest state or not.
6407 *
6408 * @remarks If you modify code here, make sure to check whether
6409 * hmR0VmxCallRing3Callback() needs to be updated too.
6410 * @remarks No-long-jmp zone!!!
6411 */
6412static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6413{
6414 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6415 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6416
6417 RTCPUID idCpu = RTMpCpuId();
6418 Log4Func(("HostCpuId=%u\n", idCpu));
6419
6420 /* Save the guest state if necessary. */
6421 if ( fSaveGuestState
6422 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6423 {
6424 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6425 AssertRCReturn(rc, rc);
6426 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6427 }
6428
6429 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6430 if (CPUMIsGuestFPUStateActive(pVCpu))
6431 {
6432 /* We shouldn't reload CR0 without saving it first. */
6433 if (!fSaveGuestState)
6434 {
6435 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6436 AssertRCReturn(rc, rc);
6437 }
6438 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6439 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6440 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6441 }
6442
6443 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6444#ifdef VBOX_STRICT
6445 if (CPUMIsHyperDebugStateActive(pVCpu))
6446 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6447#endif
6448 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6449 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6450 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6451 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6452
6453#if HC_ARCH_BITS == 64
6454 /* Restore host-state bits that VT-x only restores partially. */
6455 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6456 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6457 {
6458 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6459 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6460 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6461 }
6462#endif
6463
6464 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6465 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6466 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6467 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6468 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6469 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6470 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6471 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6472
6473 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6474
6475 /** @todo This kinda defeats the purpose of having preemption hooks.
6476 * The problem is, deregistering the hooks should be moved to a place that
6477 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6478 * context.
6479 */
6480 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6481 {
6482 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6483 AssertRCReturn(rc, rc);
6484
6485 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6486 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6487 }
6488 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6489 NOREF(idCpu);
6490
6491 return VINF_SUCCESS;
6492}
6493
6494
6495/**
6496 * Leaves the VT-x session.
6497 *
6498 * @returns VBox status code.
6499 * @param pVM Pointer to the VM.
6500 * @param pVCpu Pointer to the VMCPU.
6501 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6502 * out-of-sync. Make sure to update the required fields
6503 * before using them.
6504 *
6505 * @remarks No-long-jmp zone!!!
6506 */
6507DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6508{
6509 HM_DISABLE_PREEMPT_IF_NEEDED();
6510 HMVMX_ASSERT_CPU_SAFE();
6511 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6512 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6513
6514 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6515 and done this from the VMXR0ThreadCtxCallback(). */
6516 if (!pVCpu->hm.s.fLeaveDone)
6517 {
6518 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6519 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6520 pVCpu->hm.s.fLeaveDone = true;
6521 }
6522
6523 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6524 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6525 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6526 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6527 VMMR0ThreadCtxHooksDeregister(pVCpu);
6528
6529 /* Leave HM context. This takes care of local init (term). */
6530 int rc = HMR0LeaveCpu(pVCpu);
6531
6532 HM_RESTORE_PREEMPT_IF_NEEDED();
6533
6534 return rc;
6535}
6536
6537
6538/**
6539 * Does the necessary state syncing before doing a longjmp to ring-3.
6540 *
6541 * @returns VBox status code.
6542 * @param pVM Pointer to the VM.
6543 * @param pVCpu Pointer to the VMCPU.
6544 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6545 * out-of-sync. Make sure to update the required fields
6546 * before using them.
6547 *
6548 * @remarks No-long-jmp zone!!!
6549 */
6550DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6551{
6552 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6553}
6554
6555
6556/**
6557 * Take necessary actions before going back to ring-3.
6558 *
6559 * An action requires us to go back to ring-3. This function does the necessary
6560 * steps before we can safely return to ring-3. This is not the same as longjmps
6561 * to ring-3, this is voluntary and prepares the guest so it may continue
6562 * executing outside HM (recompiler/IEM).
6563 *
6564 * @returns VBox status code.
6565 * @param pVM Pointer to the VM.
6566 * @param pVCpu Pointer to the VMCPU.
6567 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6568 * out-of-sync. Make sure to update the required fields
6569 * before using them.
6570 * @param rcExit The reason for exiting to ring-3. Can be
6571 * VINF_VMM_UNKNOWN_RING3_CALL.
6572 */
6573static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6574{
6575 Assert(pVM);
6576 Assert(pVCpu);
6577 Assert(pMixedCtx);
6578 HMVMX_ASSERT_PREEMPT_SAFE();
6579
6580 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6581 {
6582 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6583 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6584 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6585 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6586 }
6587
6588 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6589 VMMRZCallRing3Disable(pVCpu);
6590 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6591
6592 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6593 if (pVCpu->hm.s.Event.fPending)
6594 {
6595 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6596 Assert(!pVCpu->hm.s.Event.fPending);
6597 }
6598
6599 /* Save guest state and restore host state bits. */
6600 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6601 AssertRCReturn(rc, rc);
6602 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6603
6604 /* Sync recompiler state. */
6605 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6606 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6607 | CPUM_CHANGED_LDTR
6608 | CPUM_CHANGED_GDTR
6609 | CPUM_CHANGED_IDTR
6610 | CPUM_CHANGED_TR
6611 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6612 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6613 if ( pVM->hm.s.fNestedPaging
6614 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6615 {
6616 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6617 }
6618
6619 Assert(!pVCpu->hm.s.fClearTrapFlag);
6620
6621 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6622 if (rcExit != VINF_EM_RAW_INTERRUPT)
6623 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6624
6625 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6626
6627 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6628 VMMRZCallRing3RemoveNotification(pVCpu);
6629 VMMRZCallRing3Enable(pVCpu);
6630
6631 return rc;
6632}
6633
6634
6635/**
6636 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6637 * longjump to ring-3 and possibly get preempted.
6638 *
6639 * @returns VBox status code.
6640 * @param pVCpu Pointer to the VMCPU.
6641 * @param enmOperation The operation causing the ring-3 longjump.
6642 * @param pvUser Opaque pointer to the guest-CPU context. The data
6643 * may be out-of-sync. Make sure to update the required
6644 * fields before using them.
6645 * @remarks If you modify code here, make sure to check whether
6646 * hmR0VmxLeave() needs to be updated too.
6647 */
6648DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6649{
6650 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6651 {
6652 VMMRZCallRing3RemoveNotification(pVCpu);
6653 HM_DISABLE_PREEMPT_IF_NEEDED();
6654
6655 /* If anything here asserts or fails, good luck. */
6656 if (CPUMIsGuestFPUStateActive(pVCpu))
6657 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6658
6659 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6660
6661#if HC_ARCH_BITS == 64
6662 /* Restore host-state bits that VT-x only restores partially. */
6663 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6664 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6665 {
6666 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6667 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6668 }
6669#endif
6670 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6671 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6672 {
6673 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6674 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6675 }
6676
6677 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6678 VMMR0ThreadCtxHooksDeregister(pVCpu);
6679
6680 HMR0LeaveCpu(pVCpu);
6681 HM_RESTORE_PREEMPT_IF_NEEDED();
6682 return VINF_SUCCESS;
6683 }
6684
6685 Assert(pVCpu);
6686 Assert(pvUser);
6687 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6688 HMVMX_ASSERT_PREEMPT_SAFE();
6689
6690 VMMRZCallRing3Disable(pVCpu);
6691 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6692
6693 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6694 enmOperation));
6695
6696 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6697 AssertRCReturn(rc, rc);
6698
6699 VMMRZCallRing3Enable(pVCpu);
6700 return VINF_SUCCESS;
6701}
6702
6703
6704/**
6705 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6706 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6707 *
6708 * @param pVCpu Pointer to the VMCPU.
6709 */
6710DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6711{
6712 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6713 {
6714 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6715 {
6716 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6717 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6718 AssertRC(rc);
6719 }
6720 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6721}
6722
6723
6724/**
6725 * Evaluates the event to be delivered to the guest and sets it as the pending
6726 * event.
6727 *
6728 * @param pVCpu Pointer to the VMCPU.
6729 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6730 * out-of-sync. Make sure to update the required fields
6731 * before using them.
6732 */
6733static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6734{
6735 Assert(!pVCpu->hm.s.Event.fPending);
6736
6737 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6738 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6739 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6740 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6741
6742 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6743 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6744 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6745 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6746 Assert(!TRPMHasTrap(pVCpu));
6747
6748 /** @todo SMI. SMIs take priority over NMIs. */
6749 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6750 {
6751 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6752 if ( !fBlockMovSS
6753 && !fBlockSti)
6754 {
6755 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6756 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6757 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6758 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6759
6760 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6761 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6762 }
6763 else
6764 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6765 }
6766 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6767 && !pVCpu->hm.s.fSingleInstruction)
6768 {
6769 /*
6770 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6771 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6772 * evaluated here and not set as pending, solely based on the force-flags.
6773 */
6774 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6775 AssertRC(rc);
6776 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6777 if ( !fBlockInt
6778 && !fBlockSti
6779 && !fBlockMovSS)
6780 {
6781 uint8_t u8Interrupt;
6782 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6783 if (RT_SUCCESS(rc))
6784 {
6785 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6786 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6787 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6788
6789 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6790 }
6791 else
6792 {
6793 /** @todo Does this actually happen? If not turn it into an assertion. */
6794 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6795 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6796 }
6797 }
6798 else
6799 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6800 }
6801}
6802
6803
6804/**
6805 * Injects any pending events into the guest if the guest is in a state to
6806 * receive them.
6807 *
6808 * @returns VBox status code (informational status codes included).
6809 * @param pVCpu Pointer to the VMCPU.
6810 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6811 * out-of-sync. Make sure to update the required fields
6812 * before using them.
6813 */
6814static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6815{
6816 HMVMX_ASSERT_PREEMPT_SAFE();
6817 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6818
6819 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6820 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6821 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6822 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6823
6824 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6825 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6826 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6827 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6828 Assert(!TRPMHasTrap(pVCpu));
6829
6830 int rc = VINF_SUCCESS;
6831 if (pVCpu->hm.s.Event.fPending)
6832 {
6833#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6834 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6835 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6836 {
6837 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6838 AssertRCReturn(rc, rc);
6839 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6840 Assert(!fBlockInt);
6841 Assert(!fBlockSti);
6842 Assert(!fBlockMovSS);
6843 }
6844 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6845 {
6846 Assert(!fBlockSti);
6847 Assert(!fBlockMovSS);
6848 }
6849#endif
6850 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6851 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6852 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6853 AssertRCReturn(rc, rc);
6854
6855 /* Update the interruptibility-state as it could have been changed by
6856 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6857 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6858 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6859
6860#ifdef VBOX_WITH_STATISTICS
6861 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6862 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6863 else
6864 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6865#endif
6866 }
6867
6868 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6869 int rc2 = VINF_SUCCESS;
6870 if ( fBlockSti
6871 || fBlockMovSS)
6872 {
6873 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6874 {
6875 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6876 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6877 {
6878 /*
6879 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6880 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6881 * See Intel spec. 27.3.4 "Saving Non-Register State".
6882 */
6883 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6884 AssertRCReturn(rc, rc);
6885 }
6886 }
6887 else
6888 {
6889 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6890 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6891 uIntrState = 0;
6892 }
6893 }
6894
6895 /*
6896 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6897 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6898 */
6899 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6900 AssertRC(rc2);
6901
6902 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6903 return rc;
6904}
6905
6906
6907/**
6908 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6909 *
6910 * @param pVCpu Pointer to the VMCPU.
6911 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6912 * out-of-sync. Make sure to update the required fields
6913 * before using them.
6914 */
6915DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6916{
6917 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6918 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6919}
6920
6921
6922/**
6923 * Injects a double-fault (#DF) exception into the VM.
6924 *
6925 * @returns VBox status code (informational status code included).
6926 * @param pVCpu Pointer to the VMCPU.
6927 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6928 * out-of-sync. Make sure to update the required fields
6929 * before using them.
6930 */
6931DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6932{
6933 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6934 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6935 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6936 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6937 puIntrState);
6938}
6939
6940
6941/**
6942 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6943 *
6944 * @param pVCpu Pointer to the VMCPU.
6945 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6946 * out-of-sync. Make sure to update the required fields
6947 * before using them.
6948 */
6949DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6950{
6951 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6952 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6953 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6954}
6955
6956
6957/**
6958 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6959 *
6960 * @param pVCpu Pointer to the VMCPU.
6961 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6962 * out-of-sync. Make sure to update the required fields
6963 * before using them.
6964 * @param cbInstr The value of RIP that is to be pushed on the guest
6965 * stack.
6966 */
6967DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6968{
6969 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6970 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6971 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6972}
6973
6974
6975/**
6976 * Injects a general-protection (#GP) fault into the VM.
6977 *
6978 * @returns VBox status code (informational status code included).
6979 * @param pVCpu Pointer to the VMCPU.
6980 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6981 * out-of-sync. Make sure to update the required fields
6982 * before using them.
6983 * @param u32ErrorCode The error code associated with the #GP.
6984 */
6985DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6986 uint32_t *puIntrState)
6987{
6988 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6989 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6990 if (fErrorCodeValid)
6991 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6992 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6993 puIntrState);
6994}
6995
6996
6997/**
6998 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6999 *
7000 * @param pVCpu Pointer to the VMCPU.
7001 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7002 * out-of-sync. Make sure to update the required fields
7003 * before using them.
7004 * @param uVector The software interrupt vector number.
7005 * @param cbInstr The value of RIP that is to be pushed on the guest
7006 * stack.
7007 */
7008DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7009{
7010 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7011 if ( uVector == X86_XCPT_BP
7012 || uVector == X86_XCPT_OF)
7013 {
7014 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7015 }
7016 else
7017 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7018 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7019}
7020
7021
7022/**
7023 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7024 * stack.
7025 *
7026 * @returns VBox status code (information status code included).
7027 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7028 * @param pVM Pointer to the VM.
7029 * @param pMixedCtx Pointer to the guest-CPU context.
7030 * @param uValue The value to push to the guest stack.
7031 */
7032DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7033{
7034 /*
7035 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7036 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7037 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7038 */
7039 if (pMixedCtx->sp == 1)
7040 return VINF_EM_RESET;
7041 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7042 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7043 AssertRCReturn(rc, rc);
7044 return rc;
7045}
7046
7047
7048/**
7049 * Injects an event into the guest upon VM-entry by updating the relevant fields
7050 * in the VM-entry area in the VMCS.
7051 *
7052 * @returns VBox status code (informational error codes included).
7053 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7054 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7055 *
7056 * @param pVCpu Pointer to the VMCPU.
7057 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7058 * be out-of-sync. Make sure to update the required
7059 * fields before using them.
7060 * @param u64IntInfo The VM-entry interruption-information field.
7061 * @param cbInstr The VM-entry instruction length in bytes (for
7062 * software interrupts, exceptions and privileged
7063 * software exceptions).
7064 * @param u32ErrCode The VM-entry exception error code.
7065 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7066 * @param puIntrState Pointer to the current guest interruptibility-state.
7067 * This interruptibility-state will be updated if
7068 * necessary. This cannot not be NULL.
7069 *
7070 * @remarks Requires CR0!
7071 * @remarks No-long-jump zone!!!
7072 */
7073static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7074 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7075{
7076 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7077 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7078 Assert(puIntrState);
7079 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7080
7081 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7082 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7083
7084#ifdef VBOX_STRICT
7085 /* Validate the error-code-valid bit for hardware exceptions. */
7086 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7087 {
7088 switch (uVector)
7089 {
7090 case X86_XCPT_PF:
7091 case X86_XCPT_DF:
7092 case X86_XCPT_TS:
7093 case X86_XCPT_NP:
7094 case X86_XCPT_SS:
7095 case X86_XCPT_GP:
7096 case X86_XCPT_AC:
7097 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7098 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7099 /* fallthru */
7100 default:
7101 break;
7102 }
7103 }
7104#endif
7105
7106 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7107 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7108 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7109
7110 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7111
7112 /* We require CR0 to check if the guest is in real-mode. */
7113 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7114 AssertRCReturn(rc, rc);
7115
7116 /*
7117 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7118 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7119 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7120 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7121 */
7122 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7123 {
7124 PVM pVM = pVCpu->CTX_SUFF(pVM);
7125 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7126 {
7127 Assert(PDMVmmDevHeapIsEnabled(pVM));
7128 Assert(pVM->hm.s.vmx.pRealModeTSS);
7129
7130 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7131 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7132 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7133 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7134 AssertRCReturn(rc, rc);
7135 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7136
7137 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7138 const size_t cbIdtEntry = sizeof(X86IDTR16);
7139 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7140 {
7141 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7142 if (uVector == X86_XCPT_DF)
7143 return VINF_EM_RESET;
7144 else if (uVector == X86_XCPT_GP)
7145 {
7146 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7147 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7148 }
7149
7150 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7151 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7152 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7153 }
7154
7155 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7156 uint16_t uGuestIp = pMixedCtx->ip;
7157 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7158 {
7159 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7160 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7161 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7162 }
7163 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7164 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7165
7166 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7167 X86IDTR16 IdtEntry;
7168 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7169 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7170 AssertRCReturn(rc, rc);
7171
7172 /* Construct the stack frame for the interrupt/exception handler. */
7173 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7174 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7175 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7176 AssertRCReturn(rc, rc);
7177
7178 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7179 if (rc == VINF_SUCCESS)
7180 {
7181 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7182 pMixedCtx->rip = IdtEntry.offSel;
7183 pMixedCtx->cs.Sel = IdtEntry.uSel;
7184 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7185 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7186 && uVector == X86_XCPT_PF)
7187 {
7188 pMixedCtx->cr2 = GCPtrFaultAddress;
7189 }
7190
7191 /* If any other guest-state bits are changed here, make sure to update
7192 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7193 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7194 | HM_CHANGED_GUEST_RIP
7195 | HM_CHANGED_GUEST_RFLAGS
7196 | HM_CHANGED_GUEST_RSP);
7197
7198 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7199 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7200 {
7201 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7202 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7203 Log4(("Clearing inhibition due to STI.\n"));
7204 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7205 }
7206 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7207
7208 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7209 it, if we are returning to ring-3 before executing guest code. */
7210 pVCpu->hm.s.Event.fPending = false;
7211 }
7212 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7213 return rc;
7214 }
7215 else
7216 {
7217 /*
7218 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7219 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7220 */
7221 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7222 }
7223 }
7224
7225 /* Validate. */
7226 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7227 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7228 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7229
7230 /* Inject. */
7231 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7232 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7233 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7234 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7235
7236 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7237 && uVector == X86_XCPT_PF)
7238 {
7239 pMixedCtx->cr2 = GCPtrFaultAddress;
7240 }
7241
7242 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7243 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7244
7245 AssertRCReturn(rc, rc);
7246 return rc;
7247}
7248
7249
7250/**
7251 * Clears the interrupt-window exiting control in the VMCS and if necessary
7252 * clears the current event in the VMCS as well.
7253 *
7254 * @returns VBox status code.
7255 * @param pVCpu Pointer to the VMCPU.
7256 *
7257 * @remarks Use this function only to clear events that have not yet been
7258 * delivered to the guest but are injected in the VMCS!
7259 * @remarks No-long-jump zone!!!
7260 */
7261static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7262{
7263 int rc;
7264 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7265
7266 /* Clear interrupt-window exiting control. */
7267 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7268 {
7269 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7270 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7271 AssertRC(rc);
7272 }
7273
7274 if (!pVCpu->hm.s.Event.fPending)
7275 return;
7276
7277#ifdef VBOX_STRICT
7278 uint32_t u32EntryInfo;
7279 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7280 AssertRC(rc);
7281 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7282#endif
7283
7284 /* Clear the entry-interruption field (including the valid bit). */
7285 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7286 AssertRC(rc);
7287
7288 /* Clear the pending debug exception field. */
7289 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7290 AssertRC(rc);
7291}
7292
7293
7294/**
7295 * Enters the VT-x session.
7296 *
7297 * @returns VBox status code.
7298 * @param pVM Pointer to the VM.
7299 * @param pVCpu Pointer to the VMCPU.
7300 * @param pCpu Pointer to the CPU info struct.
7301 */
7302VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7303{
7304 AssertPtr(pVM);
7305 AssertPtr(pVCpu);
7306 Assert(pVM->hm.s.vmx.fSupported);
7307 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7308 NOREF(pCpu);
7309
7310 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7311 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7312
7313#ifdef VBOX_STRICT
7314 /* Make sure we're in VMX root mode. */
7315 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7316 if (!(u32HostCR4 & X86_CR4_VMXE))
7317 {
7318 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7319 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7320 }
7321#endif
7322
7323 /*
7324 * Load the VCPU's VMCS as the current (and active) one.
7325 */
7326 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7327 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7328 if (RT_FAILURE(rc))
7329 return rc;
7330
7331 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7332 pVCpu->hm.s.fLeaveDone = false;
7333 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7334
7335 return VINF_SUCCESS;
7336}
7337
7338
7339/**
7340 * The thread-context callback (only on platforms which support it).
7341 *
7342 * @param enmEvent The thread-context event.
7343 * @param pVCpu Pointer to the VMCPU.
7344 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7345 * @thread EMT(pVCpu)
7346 */
7347VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7348{
7349 switch (enmEvent)
7350 {
7351 case RTTHREADCTXEVENT_PREEMPTING:
7352 {
7353 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7354 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7355 VMCPU_ASSERT_EMT(pVCpu);
7356
7357 PVM pVM = pVCpu->CTX_SUFF(pVM);
7358 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7359
7360 /* No longjmps (logger flushes, locks) in this fragile context. */
7361 VMMRZCallRing3Disable(pVCpu);
7362 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7363
7364 /*
7365 * Restore host-state (FPU, debug etc.)
7366 */
7367 if (!pVCpu->hm.s.fLeaveDone)
7368 {
7369 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7370 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7371 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7372 pVCpu->hm.s.fLeaveDone = true;
7373 }
7374
7375 /* Leave HM context, takes care of local init (term). */
7376 int rc = HMR0LeaveCpu(pVCpu);
7377 AssertRC(rc); NOREF(rc);
7378
7379 /* Restore longjmp state. */
7380 VMMRZCallRing3Enable(pVCpu);
7381 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7382 break;
7383 }
7384
7385 case RTTHREADCTXEVENT_RESUMED:
7386 {
7387 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7388 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7389 VMCPU_ASSERT_EMT(pVCpu);
7390
7391 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7392 VMMRZCallRing3Disable(pVCpu);
7393 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7394
7395 /* Initialize the bare minimum state required for HM. This takes care of
7396 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7397 int rc = HMR0EnterCpu(pVCpu);
7398 AssertRC(rc);
7399 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7400
7401 /* Load the active VMCS as the current one. */
7402 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7403 {
7404 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7405 AssertRC(rc); NOREF(rc);
7406 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7407 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7408 }
7409 pVCpu->hm.s.fLeaveDone = false;
7410
7411 /* Restore longjmp state. */
7412 VMMRZCallRing3Enable(pVCpu);
7413 break;
7414 }
7415
7416 default:
7417 break;
7418 }
7419}
7420
7421
7422/**
7423 * Saves the host state in the VMCS host-state.
7424 * Sets up the VM-exit MSR-load area.
7425 *
7426 * The CPU state will be loaded from these fields on every successful VM-exit.
7427 *
7428 * @returns VBox status code.
7429 * @param pVM Pointer to the VM.
7430 * @param pVCpu Pointer to the VMCPU.
7431 *
7432 * @remarks No-long-jump zone!!!
7433 */
7434static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7435{
7436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7437
7438 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7439 return VINF_SUCCESS;
7440
7441 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7442 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7443
7444 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7445 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7446
7447 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7448 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7449
7450 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7451 return rc;
7452}
7453
7454
7455/**
7456 * Saves the host state in the VMCS host-state.
7457 *
7458 * @returns VBox status code.
7459 * @param pVM Pointer to the VM.
7460 * @param pVCpu Pointer to the VMCPU.
7461 *
7462 * @remarks No-long-jump zone!!!
7463 */
7464VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7465{
7466 AssertPtr(pVM);
7467 AssertPtr(pVCpu);
7468
7469 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7470
7471 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7472 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7473 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7474 return hmR0VmxSaveHostState(pVM, pVCpu);
7475}
7476
7477
7478/**
7479 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7480 * loaded from these fields on every successful VM-entry.
7481 *
7482 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7483 * Sets up the VM-entry controls.
7484 * Sets up the appropriate VMX non-root function to execute guest code based on
7485 * the guest CPU mode.
7486 *
7487 * @returns VBox status code.
7488 * @param pVM Pointer to the VM.
7489 * @param pVCpu Pointer to the VMCPU.
7490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7491 * out-of-sync. Make sure to update the required fields
7492 * before using them.
7493 *
7494 * @remarks No-long-jump zone!!!
7495 */
7496static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7497{
7498 AssertPtr(pVM);
7499 AssertPtr(pVCpu);
7500 AssertPtr(pMixedCtx);
7501 HMVMX_ASSERT_PREEMPT_SAFE();
7502
7503#ifdef LOG_ENABLED
7504 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7505 * probably not initialized yet? Anyway this will do for now.
7506 *
7507 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7508 * interface and disable ring-3 calls when thread-context hooks are not
7509 * available. */
7510 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7511 VMMR0LogFlushDisable(pVCpu);
7512#endif
7513
7514 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7515
7516 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7517
7518 /* Determine real-on-v86 mode. */
7519 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7520 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7521 && CPUMIsGuestInRealModeEx(pMixedCtx))
7522 {
7523 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7524 }
7525
7526 /*
7527 * Load the guest-state into the VMCS.
7528 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7529 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7530 */
7531 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7532 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7533
7534 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7535 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7536 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7537
7538 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7539 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7540 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7541
7542 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7543 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7544
7545 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7546 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7547
7548 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7549 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7550 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7551
7552 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7553 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7554
7555 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7556 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7557
7558 /*
7559 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7560 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7561 */
7562 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7563 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7564
7565 /* Clear any unused and reserved bits. */
7566 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7567
7568#ifdef LOG_ENABLED
7569 /* Only reenable log-flushing if the caller has it enabled. */
7570 if (!fCallerDisabledLogFlush)
7571 VMMR0LogFlushEnable(pVCpu);
7572#endif
7573
7574 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7575 return rc;
7576}
7577
7578
7579/**
7580 * Loads the state shared between the host and guest into the VMCS.
7581 *
7582 * @param pVM Pointer to the VM.
7583 * @param pVCpu Pointer to the VMCPU.
7584 * @param pCtx Pointer to the guest-CPU context.
7585 *
7586 * @remarks No-long-jump zone!!!
7587 */
7588static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7589{
7590 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7591 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7592
7593 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7594 {
7595 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7596 AssertRC(rc);
7597 }
7598
7599 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7600 {
7601 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7602 AssertRC(rc);
7603
7604 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7605 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7606 {
7607 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7608 AssertRC(rc);
7609 }
7610 }
7611
7612 AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7613 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7614}
7615
7616
7617/**
7618 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7619 *
7620 * @param pVM Pointer to the VM.
7621 * @param pVCpu Pointer to the VMCPU.
7622 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7623 * out-of-sync. Make sure to update the required fields
7624 * before using them.
7625 */
7626DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7627{
7628 HMVMX_ASSERT_PREEMPT_SAFE();
7629
7630 Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7631#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7632 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7633#endif
7634
7635 if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7636 {
7637 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7638 AssertRC(rc);
7639 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7640 }
7641 else if (VMCPU_HMCF_VALUE(pVCpu))
7642 {
7643 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7644 AssertRC(rc);
7645 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7646 }
7647
7648 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7649 AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7650 || VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
7651 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7652
7653#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7654 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7655 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7656 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7657#endif
7658}
7659
7660
7661/**
7662 * Does the preparations before executing guest code in VT-x.
7663 *
7664 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7665 * recompiler. We must be cautious what we do here regarding committing
7666 * guest-state information into the VMCS assuming we assuredly execute the
7667 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7668 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7669 * so that the recompiler can (and should) use them when it resumes guest
7670 * execution. Otherwise such operations must be done when we can no longer
7671 * exit to ring-3.
7672 *
7673 * @returns Strict VBox status code.
7674 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7675 * have been disabled.
7676 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7677 * double-fault into the guest.
7678 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7679 *
7680 * @param pVM Pointer to the VM.
7681 * @param pVCpu Pointer to the VMCPU.
7682 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7683 * out-of-sync. Make sure to update the required fields
7684 * before using them.
7685 * @param pVmxTransient Pointer to the VMX transient structure.
7686 *
7687 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7688 * interrupts will be disabled.
7689 */
7690static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7691{
7692 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7693
7694#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7695 PGMRZDynMapFlushAutoSet(pVCpu);
7696#endif
7697
7698 /* Check force flag actions that might require us to go back to ring-3. */
7699 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7700 if (rc != VINF_SUCCESS)
7701 return rc;
7702
7703#ifndef IEM_VERIFICATION_MODE_FULL
7704 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7705 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7706 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7707 {
7708 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7709 RTGCPHYS GCPhysApicBase;
7710 GCPhysApicBase = pMixedCtx->msrApicBase;
7711 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7712
7713 /* Unalias any existing mapping. */
7714 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7715 AssertRCReturn(rc, rc);
7716
7717 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7718 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7719 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7720 AssertRCReturn(rc, rc);
7721
7722 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7723 }
7724#endif /* !IEM_VERIFICATION_MODE_FULL */
7725
7726 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7727 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7728
7729 /*
7730 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7731 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7732 */
7733 if (TRPMHasTrap(pVCpu))
7734 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7735 else if (!pVCpu->hm.s.Event.fPending)
7736 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7737
7738 /*
7739 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7740 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7741 */
7742 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7743 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7744 {
7745 Assert(rc == VINF_EM_RESET);
7746 return rc;
7747 }
7748
7749 /*
7750 * No longjmps to ring-3 from this point on!!!
7751 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7752 * This also disables flushing of the R0-logger instance (if any).
7753 */
7754 VMMRZCallRing3Disable(pVCpu);
7755
7756 /*
7757 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7758 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7759 *
7760 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7761 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7762 *
7763 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7764 * executing guest code.
7765 */
7766 pVmxTransient->uEflags = ASMIntDisableFlags();
7767 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7768 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7769 {
7770 hmR0VmxClearEventVmcs(pVCpu);
7771 ASMSetFlags(pVmxTransient->uEflags);
7772 VMMRZCallRing3Enable(pVCpu);
7773 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7774 return VINF_EM_RAW_TO_R3;
7775 }
7776 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7777 {
7778 hmR0VmxClearEventVmcs(pVCpu);
7779 ASMSetFlags(pVmxTransient->uEflags);
7780 VMMRZCallRing3Enable(pVCpu);
7781 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7782 return VINF_EM_RAW_INTERRUPT;
7783 }
7784
7785 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7786 pVCpu->hm.s.Event.fPending = false;
7787
7788 return VINF_SUCCESS;
7789}
7790
7791
7792/**
7793 * Prepares to run guest code in VT-x and we've committed to doing so. This
7794 * means there is no backing out to ring-3 or anywhere else at this
7795 * point.
7796 *
7797 * @param pVM Pointer to the VM.
7798 * @param pVCpu Pointer to the VMCPU.
7799 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7800 * out-of-sync. Make sure to update the required fields
7801 * before using them.
7802 * @param pVmxTransient Pointer to the VMX transient structure.
7803 *
7804 * @remarks Called with preemption disabled.
7805 * @remarks No-long-jump zone!!!
7806 */
7807static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7808{
7809 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7810 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7811 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7812
7813 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7814 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7815
7816 /*
7817 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7818 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7819 * Reload only the necessary state, the assertion will catch if other parts of the code
7820 * change.
7821 */
7822 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7823 {
7824 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7825 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7826 }
7827
7828#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7829 if (!CPUMIsGuestFPUStateActive(pVCpu))
7830 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7831 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7832#endif
7833
7834 /*
7835 * Load the host state bits as we may've been preempted (only happens when
7836 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7837 */
7838 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7839 {
7840 /* This ASSUMES that pfnStartVM has been set up already. */
7841 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7842 AssertRC(rc);
7843 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7844 }
7845 Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7846
7847 /*
7848 * Load the state shared between host and guest (FPU, debug).
7849 */
7850 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7851 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7852 AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7853
7854 /* Store status of the shared guest-host state at the time of VM-entry. */
7855#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7856 if (CPUMIsGuestInLongModeEx(pMixedCtx))
7857 {
7858 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7859 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7860 }
7861 else
7862#endif
7863 {
7864 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7865 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7866 }
7867 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7868
7869 /*
7870 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7871 */
7872 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7873 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7874
7875 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7876 RTCPUID idCurrentCpu = pCpu->idCpu;
7877 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7878 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7879 {
7880 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7881 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7882 }
7883
7884 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7885 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7886 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7887 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7888
7889 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7890
7891 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7892 to start executing. */
7893
7894#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7895 /*
7896 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7897 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7898 */
7899 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7900 {
7901 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7902 uint64_t u64HostTscAux = 0;
7903 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7904 AssertRC(rc2);
7905 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7906 }
7907#endif
7908}
7909
7910
7911/**
7912 * Performs some essential restoration of state after running guest code in
7913 * VT-x.
7914 *
7915 * @param pVM Pointer to the VM.
7916 * @param pVCpu Pointer to the VMCPU.
7917 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7918 * out-of-sync. Make sure to update the required fields
7919 * before using them.
7920 * @param pVmxTransient Pointer to the VMX transient structure.
7921 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7922 *
7923 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7924 *
7925 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7926 * unconditionally when it is safe to do so.
7927 */
7928static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7929{
7930 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7931
7932 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7933 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7934 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7935 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7936 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7937
7938 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7939 {
7940#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7941 /* Restore host's TSC_AUX. */
7942 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7943 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7944#endif
7945 /** @todo Find a way to fix hardcoding a guestimate. */
7946 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7947 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7948 }
7949
7950 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7951 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7952 Assert(!(ASMGetFlags() & X86_EFL_IF));
7953 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7954
7955#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7956 if (CPUMIsGuestFPUStateActive(pVCpu))
7957 {
7958 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7959 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7960 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7961 }
7962#endif
7963
7964 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7965 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7966 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7967 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7968
7969 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7970 uint32_t uExitReason;
7971 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7972 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7973 AssertRC(rc);
7974 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7975 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntInfo);
7976
7977 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7978 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7979 {
7980 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7981 pVmxTransient->fVMEntryFailed));
7982 return;
7983 }
7984
7985 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7986 {
7987 /* Update the guest interruptibility-state from the VMCS. */
7988 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7989#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7990 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7991 AssertRC(rc);
7992#endif
7993 /*
7994 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7995 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7996 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7997 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7998 */
7999 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8000 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8001 {
8002 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8003 AssertRC(rc);
8004 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8005 }
8006 }
8007}
8008
8009
8010
8011/**
8012 * Runs the guest code using VT-x the normal way.
8013 *
8014 * @returns VBox status code.
8015 * @param pVM Pointer to the VM.
8016 * @param pVCpu Pointer to the VMCPU.
8017 * @param pCtx Pointer to the guest-CPU context.
8018 *
8019 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
8020 * @remarks Called with preemption disabled.
8021 */
8022static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8023{
8024 VMXTRANSIENT VmxTransient;
8025 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8026 int rc = VERR_INTERNAL_ERROR_5;
8027 uint32_t cLoops = 0;
8028
8029 for (;; cLoops++)
8030 {
8031 Assert(!HMR0SuspendPending());
8032 HMVMX_ASSERT_CPU_SAFE();
8033
8034 /* Preparatory work for running guest code, this may force us to return
8035 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8036 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8037 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8038 if (rc != VINF_SUCCESS)
8039 break;
8040
8041 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8042 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8043 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8044
8045 /* Restore any residual host-state and save any bits shared between host
8046 and guest into the guest-CPU state. Re-enables interrupts! */
8047 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8048
8049 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8050 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8051 {
8052 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8053 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8054 return rc;
8055 }
8056
8057 /* Handle the VM-exit. */
8058 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8059 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8060 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8061 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8062 HMVMX_START_EXIT_DISPATCH_PROF();
8063#ifdef HMVMX_USE_FUNCTION_TABLE
8064 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8065#else
8066 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8067#endif
8068 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8069 if (rc != VINF_SUCCESS)
8070 break;
8071 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8072 {
8073 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8074 rc = VINF_EM_RAW_INTERRUPT;
8075 break;
8076 }
8077 }
8078
8079 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8080 return rc;
8081}
8082
8083
8084/**
8085 * Single steps guest code using VT-x.
8086 *
8087 * @returns VBox status code.
8088 * @param pVM Pointer to the VM.
8089 * @param pVCpu Pointer to the VMCPU.
8090 * @param pCtx Pointer to the guest-CPU context.
8091 *
8092 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8093 * @remarks Called with preemption disabled.
8094 */
8095static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8096{
8097 VMXTRANSIENT VmxTransient;
8098 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8099 int rc = VERR_INTERNAL_ERROR_5;
8100 uint32_t cLoops = 0;
8101 uint16_t uCsStart = pCtx->cs.Sel;
8102 uint64_t uRipStart = pCtx->rip;
8103
8104 for (;; cLoops++)
8105 {
8106 Assert(!HMR0SuspendPending());
8107 HMVMX_ASSERT_CPU_SAFE();
8108
8109 /* Preparatory work for running guest code, this may force us to return
8110 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8111 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8112 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8113 if (rc != VINF_SUCCESS)
8114 break;
8115
8116 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8117 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8118 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8119
8120 /* Restore any residual host-state and save any bits shared between host
8121 and guest into the guest-CPU state. Re-enables interrupts! */
8122 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8123
8124 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8125 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8126 {
8127 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8128 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8129 return rc;
8130 }
8131
8132 /* Handle the VM-exit. */
8133 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8135 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8136 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8137 HMVMX_START_EXIT_DISPATCH_PROF();
8138#ifdef HMVMX_USE_FUNCTION_TABLE
8139 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8140#else
8141 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8142#endif
8143 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8144 if (rc != VINF_SUCCESS)
8145 break;
8146 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8147 {
8148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8149 rc = VINF_EM_RAW_INTERRUPT;
8150 break;
8151 }
8152
8153 /*
8154 * Did the RIP change, if so, consider it a single step.
8155 * Otherwise, make sure one of the TFs gets set.
8156 */
8157 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8158 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8159 AssertRCReturn(rc2, rc2);
8160 if ( pCtx->rip != uRipStart
8161 || pCtx->cs.Sel != uCsStart)
8162 {
8163 rc = VINF_EM_DBG_STEPPED;
8164 break;
8165 }
8166 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8167 }
8168
8169 /*
8170 * Clear the X86_EFL_TF if necessary.
8171 */
8172 if (pVCpu->hm.s.fClearTrapFlag)
8173 {
8174 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8175 AssertRCReturn(rc2, rc2);
8176 pVCpu->hm.s.fClearTrapFlag = false;
8177 pCtx->eflags.Bits.u1TF = 0;
8178 }
8179 /** @todo there seems to be issues with the resume flag when the monitor trap
8180 * flag is pending without being used. Seen early in bios init when
8181 * accessing APIC page in prot mode. */
8182
8183 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8184 return rc;
8185}
8186
8187
8188/**
8189 * Runs the guest code using VT-x.
8190 *
8191 * @returns VBox status code.
8192 * @param pVM Pointer to the VM.
8193 * @param pVCpu Pointer to the VMCPU.
8194 * @param pCtx Pointer to the guest-CPU context.
8195 *
8196 * @remarks Called with preemption disabled.
8197 */
8198VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8199{
8200 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8201 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8202 HMVMX_ASSERT_PREEMPT_SAFE();
8203
8204 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8205
8206 int rc;
8207 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8208 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8209 else
8210 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8211
8212 if (rc == VERR_EM_INTERPRETER)
8213 rc = VINF_EM_RAW_EMULATE_INSTR;
8214 else if (rc == VINF_EM_RESET)
8215 rc = VINF_EM_TRIPLE_FAULT;
8216
8217 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8218 if (RT_FAILURE(rc2))
8219 {
8220 pVCpu->hm.s.u32HMError = rc;
8221 rc = rc2;
8222 }
8223 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8224 return rc;
8225}
8226
8227
8228#ifndef HMVMX_USE_FUNCTION_TABLE
8229DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8230{
8231 int rc;
8232 switch (rcReason)
8233 {
8234 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8235 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8236 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8237 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8238 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8239 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8240 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8241 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8242 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8243 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8244 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8245 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8246 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8247 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8248 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8249 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8250 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8251 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8252 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8253 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8254 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8255 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8256 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8257 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8258 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8259 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8260 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8261 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8262 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8263 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8264 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8265 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8266 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8267
8268 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8269 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8270 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8271 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8272 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8273 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8274 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8275 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8276 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8277
8278 case VMX_EXIT_VMCALL:
8279 case VMX_EXIT_VMCLEAR:
8280 case VMX_EXIT_VMLAUNCH:
8281 case VMX_EXIT_VMPTRLD:
8282 case VMX_EXIT_VMPTRST:
8283 case VMX_EXIT_VMREAD:
8284 case VMX_EXIT_VMRESUME:
8285 case VMX_EXIT_VMWRITE:
8286 case VMX_EXIT_VMXOFF:
8287 case VMX_EXIT_VMXON:
8288 case VMX_EXIT_INVEPT:
8289 case VMX_EXIT_INVVPID:
8290 case VMX_EXIT_VMFUNC:
8291 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8292 break;
8293 default:
8294 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8295 break;
8296 }
8297 return rc;
8298}
8299#endif
8300
8301#ifdef DEBUG
8302/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8303# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8304 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8305
8306# define HMVMX_ASSERT_PREEMPT_CPUID() \
8307 do \
8308 { \
8309 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8310 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8311 } while (0)
8312
8313# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8314 do { \
8315 AssertPtr(pVCpu); \
8316 AssertPtr(pMixedCtx); \
8317 AssertPtr(pVmxTransient); \
8318 Assert(pVmxTransient->fVMEntryFailed == false); \
8319 Assert(ASMIntAreEnabled()); \
8320 HMVMX_ASSERT_PREEMPT_SAFE(); \
8321 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8322 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8323 HMVMX_ASSERT_PREEMPT_SAFE(); \
8324 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8325 HMVMX_ASSERT_PREEMPT_CPUID(); \
8326 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8327 } while (0)
8328
8329# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8330 do { \
8331 Log4Func(("\n")); \
8332 } while(0)
8333#else /* Release builds */
8334# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8335# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8336#endif
8337
8338
8339/**
8340 * Advances the guest RIP after reading it from the VMCS.
8341 *
8342 * @returns VBox status code.
8343 * @param pVCpu Pointer to the VMCPU.
8344 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8345 * out-of-sync. Make sure to update the required fields
8346 * before using them.
8347 * @param pVmxTransient Pointer to the VMX transient structure.
8348 *
8349 * @remarks No-long-jump zone!!!
8350 */
8351DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8352{
8353 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8354 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8355 AssertRCReturn(rc, rc);
8356
8357 pMixedCtx->rip += pVmxTransient->cbInstr;
8358 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8359 return rc;
8360}
8361
8362
8363/**
8364 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8365 * and update error record fields accordingly.
8366 *
8367 * @return VMX_IGS_* return codes.
8368 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8369 * wrong with the guest state.
8370 *
8371 * @param pVM Pointer to the VM.
8372 * @param pVCpu Pointer to the VMCPU.
8373 * @param pCtx Pointer to the guest-CPU state.
8374 */
8375static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8376{
8377#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8378#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8379 uError = (err); \
8380 break; \
8381 } else do {} while (0)
8382/* Duplicate of IEM_IS_CANONICAL(). */
8383#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8384
8385 int rc;
8386 uint32_t uError = VMX_IGS_ERROR;
8387 uint32_t u32Val;
8388 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8389
8390 do
8391 {
8392 /*
8393 * CR0.
8394 */
8395 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8396 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8397 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8398 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8399 if (fUnrestrictedGuest)
8400 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8401
8402 uint32_t u32GuestCR0;
8403 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8404 AssertRCBreak(rc);
8405 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8406 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8407 if ( !fUnrestrictedGuest
8408 && (u32GuestCR0 & X86_CR0_PG)
8409 && !(u32GuestCR0 & X86_CR0_PE))
8410 {
8411 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8412 }
8413
8414 /*
8415 * CR4.
8416 */
8417 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8418 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8419
8420 uint32_t u32GuestCR4;
8421 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8422 AssertRCBreak(rc);
8423 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8424 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8425
8426 /*
8427 * IA32_DEBUGCTL MSR.
8428 */
8429 uint64_t u64Val;
8430 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8431 AssertRCBreak(rc);
8432 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8433 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8434 {
8435 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8436 }
8437 uint64_t u64DebugCtlMsr = u64Val;
8438
8439#ifdef VBOX_STRICT
8440 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8441 AssertRCBreak(rc);
8442 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8443#endif
8444 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8445
8446 /*
8447 * RIP and RFLAGS.
8448 */
8449 uint32_t u32Eflags;
8450#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8451 if (HMVMX_IS_64BIT_HOST_MODE())
8452 {
8453 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8454 AssertRCBreak(rc);
8455 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8456 if ( !fLongModeGuest
8457 || !pCtx->cs.Attr.n.u1Long)
8458 {
8459 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8460 }
8461 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8462 * must be identical if the "IA32e mode guest" VM-entry control is 1
8463 * and CS.L is 1. No check applies if the CPU supports 64
8464 * linear-address bits. */
8465
8466 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8467 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8468 AssertRCBreak(rc);
8469 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8470 VMX_IGS_RFLAGS_RESERVED);
8471 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8472 u32Eflags = u64Val;
8473 }
8474 else
8475#endif
8476 {
8477 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8478 AssertRCBreak(rc);
8479 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8480 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8481 }
8482
8483 if ( fLongModeGuest
8484 || ( fUnrestrictedGuest
8485 && !(u32GuestCR0 & X86_CR0_PE)))
8486 {
8487 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8488 }
8489
8490 uint32_t u32EntryInfo;
8491 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8492 AssertRCBreak(rc);
8493 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8494 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8495 {
8496 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8497 }
8498
8499 /*
8500 * 64-bit checks.
8501 */
8502#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8503 if (HMVMX_IS_64BIT_HOST_MODE())
8504 {
8505 if ( fLongModeGuest
8506 && !fUnrestrictedGuest)
8507 {
8508 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8509 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8510 }
8511
8512 if ( !fLongModeGuest
8513 && (u32GuestCR4 & X86_CR4_PCIDE))
8514 {
8515 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8516 }
8517
8518 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8519 * 51:32 beyond the processor's physical-address width are 0. */
8520
8521 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8522 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8523 {
8524 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8525 }
8526
8527 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8528 AssertRCBreak(rc);
8529 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8530
8531 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8532 AssertRCBreak(rc);
8533 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8534 }
8535#endif
8536
8537 /*
8538 * PERF_GLOBAL MSR.
8539 */
8540 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8541 {
8542 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8543 AssertRCBreak(rc);
8544 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8545 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8546 }
8547
8548 /*
8549 * PAT MSR.
8550 */
8551 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8552 {
8553 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8554 AssertRCBreak(rc);
8555 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8556 for (unsigned i = 0; i < 8; i++)
8557 {
8558 uint8_t u8Val = (u64Val & 0x7);
8559 if ( u8Val != 0 /* UC */
8560 || u8Val != 1 /* WC */
8561 || u8Val != 4 /* WT */
8562 || u8Val != 5 /* WP */
8563 || u8Val != 6 /* WB */
8564 || u8Val != 7 /* UC- */)
8565 {
8566 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8567 }
8568 u64Val >>= 3;
8569 }
8570 }
8571
8572 /*
8573 * EFER MSR.
8574 */
8575 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8576 {
8577 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8578 AssertRCBreak(rc);
8579 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8580 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8581 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8582 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8583 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8584 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8585 }
8586
8587 /*
8588 * Segment registers.
8589 */
8590 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8591 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8592 if (!(u32Eflags & X86_EFL_VM))
8593 {
8594 /* CS */
8595 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8596 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8597 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8598 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8599 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8600 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8601 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8602 /* CS cannot be loaded with NULL in protected mode. */
8603 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8604 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8605 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8606 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8607 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8608 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8609 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8610 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8611 else
8612 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8613
8614 /* SS */
8615 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8616 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8617 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8618 if ( !(pCtx->cr0 & X86_CR0_PE)
8619 || pCtx->cs.Attr.n.u4Type == 3)
8620 {
8621 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8622 }
8623 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8624 {
8625 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8626 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8627 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8628 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8629 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8630 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8631 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8632 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8633 }
8634
8635 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8636 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8637 {
8638 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8639 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8640 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8641 || pCtx->ds.Attr.n.u4Type > 11
8642 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8643 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8644 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8645 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8646 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8647 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8648 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8649 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8650 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8651 }
8652 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8653 {
8654 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8655 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8656 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8657 || pCtx->es.Attr.n.u4Type > 11
8658 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8659 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8660 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8661 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8662 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8663 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8664 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8665 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8666 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8667 }
8668 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8669 {
8670 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8671 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8672 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8673 || pCtx->fs.Attr.n.u4Type > 11
8674 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8675 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8676 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8677 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8678 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8679 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8680 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8681 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8682 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8683 }
8684 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8685 {
8686 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8687 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8688 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8689 || pCtx->gs.Attr.n.u4Type > 11
8690 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8691 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8692 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8693 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8694 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8695 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8696 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8697 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8698 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8699 }
8700 /* 64-bit capable CPUs. */
8701#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8702 if (HMVMX_IS_64BIT_HOST_MODE())
8703 {
8704 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8705 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8706 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8707 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8708 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8709 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8710 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8711 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8712 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8713 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8714 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8715 }
8716#endif
8717 }
8718 else
8719 {
8720 /* V86 mode checks. */
8721 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8722 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8723 {
8724 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8725 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8726 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8727 }
8728 else
8729 {
8730 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8731 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8732 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8733 }
8734
8735 /* CS */
8736 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8737 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8738 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8739 /* SS */
8740 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8741 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8742 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8743 /* DS */
8744 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8745 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8746 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8747 /* ES */
8748 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8749 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8750 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8751 /* FS */
8752 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8753 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8754 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8755 /* GS */
8756 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8757 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8758 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8759 /* 64-bit capable CPUs. */
8760#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8761 if (HMVMX_IS_64BIT_HOST_MODE())
8762 {
8763 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8764 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8765 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8766 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8767 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8768 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8769 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8770 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8771 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8772 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8773 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8774 }
8775#endif
8776 }
8777
8778 /*
8779 * TR.
8780 */
8781 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8782 /* 64-bit capable CPUs. */
8783#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8784 if (HMVMX_IS_64BIT_HOST_MODE())
8785 {
8786 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8787 }
8788#endif
8789 if (fLongModeGuest)
8790 {
8791 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8792 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8793 }
8794 else
8795 {
8796 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8797 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8798 VMX_IGS_TR_ATTR_TYPE_INVALID);
8799 }
8800 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8801 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8802 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8803 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8804 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8805 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8806 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8807 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8808
8809 /*
8810 * GDTR and IDTR.
8811 */
8812#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8813 if (HMVMX_IS_64BIT_HOST_MODE())
8814 {
8815 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8816 AssertRCBreak(rc);
8817 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8818
8819 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8820 AssertRCBreak(rc);
8821 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8822 }
8823#endif
8824
8825 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8826 AssertRCBreak(rc);
8827 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8828
8829 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8830 AssertRCBreak(rc);
8831 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8832
8833 /*
8834 * Guest Non-Register State.
8835 */
8836 /* Activity State. */
8837 uint32_t u32ActivityState;
8838 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8839 AssertRCBreak(rc);
8840 HMVMX_CHECK_BREAK( !u32ActivityState
8841 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8842 VMX_IGS_ACTIVITY_STATE_INVALID);
8843 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8844 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8845 uint32_t u32IntrState;
8846 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8847 AssertRCBreak(rc);
8848 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8849 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8850 {
8851 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8852 }
8853
8854 /** @todo Activity state and injecting interrupts. Left as a todo since we
8855 * currently don't use activity states but ACTIVE. */
8856
8857 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8858 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8859
8860 /* Guest interruptibility-state. */
8861 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8862 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8863 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8864 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8865 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8866 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8867 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8868 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8869 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8870 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8871 {
8872 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8873 {
8874 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8875 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8876 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8877 }
8878 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8879 {
8880 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8881 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8882 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8883 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8884 }
8885 }
8886 /** @todo Assumes the processor is not in SMM. */
8887 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8888 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8889 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8890 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8891 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8892 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8893 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8894 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8895 {
8896 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8897 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8898 }
8899
8900 /* Pending debug exceptions. */
8901 if (HMVMX_IS_64BIT_HOST_MODE())
8902 {
8903 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8904 AssertRCBreak(rc);
8905 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8906 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8907 u32Val = u64Val; /* For pending debug exceptions checks below. */
8908 }
8909 else
8910 {
8911 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8912 AssertRCBreak(rc);
8913 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8914 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8915 }
8916
8917 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8918 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8919 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8920 {
8921 if ( (u32Eflags & X86_EFL_TF)
8922 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8923 {
8924 /* Bit 14 is PendingDebug.BS. */
8925 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8926 }
8927 if ( !(u32Eflags & X86_EFL_TF)
8928 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8929 {
8930 /* Bit 14 is PendingDebug.BS. */
8931 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8932 }
8933 }
8934
8935 /* VMCS link pointer. */
8936 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8937 AssertRCBreak(rc);
8938 if (u64Val != UINT64_C(0xffffffffffffffff))
8939 {
8940 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8941 /** @todo Bits beyond the processor's physical-address width MBZ. */
8942 /** @todo 32-bit located in memory referenced by value of this field (as a
8943 * physical address) must contain the processor's VMCS revision ID. */
8944 /** @todo SMM checks. */
8945 }
8946
8947 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8948
8949 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8950 if (uError == VMX_IGS_ERROR)
8951 uError = VMX_IGS_REASON_NOT_FOUND;
8952 } while (0);
8953
8954 pVCpu->hm.s.u32HMError = uError;
8955 return uError;
8956
8957#undef HMVMX_ERROR_BREAK
8958#undef HMVMX_CHECK_BREAK
8959#undef HMVMX_IS_CANONICAL
8960}
8961
8962/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8963/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8964/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8965
8966/** @name VM-exit handlers.
8967 * @{
8968 */
8969
8970/**
8971 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8972 */
8973HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8974{
8975 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8976 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8977 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8978#if HC_ARCH_BITS == 64
8979 Assert(ASMIntAreEnabled());
8980 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8981 return VINF_SUCCESS;
8982#endif
8983 return VINF_EM_RAW_INTERRUPT;
8984}
8985
8986
8987/**
8988 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8989 */
8990HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8991{
8992 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8993 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8994
8995 int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8996 AssertRCReturn(rc, rc);
8997
8998 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
8999 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9000 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9001 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9002
9003 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9004 {
9005 /*
9006 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9007 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9008 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9009 *
9010 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9011 */
9012 VMXDispatchHostNmi();
9013 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9014 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9015 return VINF_SUCCESS;
9016 }
9017
9018 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9019 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9020 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9021 {
9022 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9023 return VINF_SUCCESS;
9024 }
9025 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9026 {
9027 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9028 return rc;
9029 }
9030
9031 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9032 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9033 switch (uIntType)
9034 {
9035 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9036 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
9037 /* no break */
9038 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9039 {
9040 switch (uVector)
9041 {
9042 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9043 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9044 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9045 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9046 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9047 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9048#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9049 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9050 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9051 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9052 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9053 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9054 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9055 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9056 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9057 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9058 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9059#endif
9060 default:
9061 {
9062 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9063 AssertRCReturn(rc, rc);
9064
9065 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9066 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9067 {
9068 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9069 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9070 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9071
9072 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9073 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
9074 AssertRCReturn(rc, rc);
9075 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9076 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9077 0 /* GCPtrFaultAddress */);
9078 AssertRCReturn(rc, rc);
9079 }
9080 else
9081 {
9082 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9083 pVCpu->hm.s.u32HMError = uVector;
9084 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9085 }
9086 break;
9087 }
9088 }
9089 break;
9090 }
9091
9092 default:
9093 {
9094 pVCpu->hm.s.u32HMError = uExitIntInfo;
9095 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9096 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9097 break;
9098 }
9099 }
9100 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9101 return rc;
9102}
9103
9104
9105/**
9106 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9107 */
9108HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9109{
9110 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9111
9112 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9113 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9114 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9115 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9116 AssertRCReturn(rc, rc);
9117
9118 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9119 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9120 return VINF_SUCCESS;
9121}
9122
9123
9124/**
9125 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9126 */
9127HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9128{
9129 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9130 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9131 HMVMX_RETURN_UNEXPECTED_EXIT();
9132}
9133
9134
9135/**
9136 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9137 */
9138HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9139{
9140 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9141 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9142 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9143}
9144
9145
9146/**
9147 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9148 */
9149HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9150{
9151 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9152 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9153 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9154}
9155
9156
9157/**
9158 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9159 */
9160HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9161{
9162 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9163 PVM pVM = pVCpu->CTX_SUFF(pVM);
9164 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9165 if (RT_LIKELY(rc == VINF_SUCCESS))
9166 {
9167 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9168 Assert(pVmxTransient->cbInstr == 2);
9169 }
9170 else
9171 {
9172 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9173 rc = VERR_EM_INTERPRETER;
9174 }
9175 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9176 return rc;
9177}
9178
9179
9180/**
9181 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9182 */
9183HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9184{
9185 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9186 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9187 AssertRCReturn(rc, rc);
9188
9189 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9190 return VINF_EM_RAW_EMULATE_INSTR;
9191
9192 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9193 HMVMX_RETURN_UNEXPECTED_EXIT();
9194}
9195
9196
9197/**
9198 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9199 */
9200HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9201{
9202 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9203 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9204 AssertRCReturn(rc, rc);
9205
9206 PVM pVM = pVCpu->CTX_SUFF(pVM);
9207 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9208 if (RT_LIKELY(rc == VINF_SUCCESS))
9209 {
9210 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9211 Assert(pVmxTransient->cbInstr == 2);
9212 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9213 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9214 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9215 }
9216 else
9217 {
9218 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9219 rc = VERR_EM_INTERPRETER;
9220 }
9221 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9222 return rc;
9223}
9224
9225
9226/**
9227 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9228 */
9229HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9230{
9231 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9232 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9233 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9234 AssertRCReturn(rc, rc);
9235
9236 PVM pVM = pVCpu->CTX_SUFF(pVM);
9237 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9238 if (RT_LIKELY(rc == VINF_SUCCESS))
9239 {
9240 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9241 Assert(pVmxTransient->cbInstr == 3);
9242 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9243 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9244 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9245 }
9246 else
9247 {
9248 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9249 rc = VERR_EM_INTERPRETER;
9250 }
9251 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9252 return rc;
9253}
9254
9255
9256/**
9257 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9258 */
9259HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9260{
9261 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9262 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9263 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9264 AssertRCReturn(rc, rc);
9265
9266 PVM pVM = pVCpu->CTX_SUFF(pVM);
9267 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9268 if (RT_LIKELY(rc == VINF_SUCCESS))
9269 {
9270 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9271 Assert(pVmxTransient->cbInstr == 2);
9272 }
9273 else
9274 {
9275 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9276 rc = VERR_EM_INTERPRETER;
9277 }
9278 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9279 return rc;
9280}
9281
9282
9283/**
9284 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9285 */
9286HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9287{
9288 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9289 PVM pVM = pVCpu->CTX_SUFF(pVM);
9290 Assert(!pVM->hm.s.fNestedPaging);
9291
9292 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9293 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9294 AssertRCReturn(rc, rc);
9295
9296 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9297 rc = VBOXSTRICTRC_VAL(rc2);
9298 if (RT_LIKELY(rc == VINF_SUCCESS))
9299 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9300 else
9301 {
9302 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9303 pVmxTransient->uExitQualification, rc));
9304 }
9305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9306 return rc;
9307}
9308
9309
9310/**
9311 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9312 */
9313HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9314{
9315 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9316 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9317 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9318 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9319 AssertRCReturn(rc, rc);
9320
9321 PVM pVM = pVCpu->CTX_SUFF(pVM);
9322 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9323 if (RT_LIKELY(rc == VINF_SUCCESS))
9324 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9325 else
9326 {
9327 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9328 rc = VERR_EM_INTERPRETER;
9329 }
9330 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9331 return rc;
9332}
9333
9334
9335/**
9336 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9337 */
9338HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9339{
9340 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9341 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9342 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9343 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9344 AssertRCReturn(rc, rc);
9345
9346 PVM pVM = pVCpu->CTX_SUFF(pVM);
9347 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9348 rc = VBOXSTRICTRC_VAL(rc2);
9349 if (RT_LIKELY( rc == VINF_SUCCESS
9350 || rc == VINF_EM_HALT))
9351 {
9352 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9353 AssertRCReturn(rc3, rc3);
9354
9355 if ( rc == VINF_EM_HALT
9356 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9357 {
9358 rc = VINF_SUCCESS;
9359 }
9360 }
9361 else
9362 {
9363 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9364 rc = VERR_EM_INTERPRETER;
9365 }
9366 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9367 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9369 return rc;
9370}
9371
9372
9373/**
9374 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9375 */
9376HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9377{
9378 /*
9379 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9380 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9381 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9382 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9383 */
9384 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9385 HMVMX_RETURN_UNEXPECTED_EXIT();
9386}
9387
9388
9389/**
9390 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9391 */
9392HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9393{
9394 /*
9395 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9396 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9397 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9398 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9399 */
9400 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9401 HMVMX_RETURN_UNEXPECTED_EXIT();
9402}
9403
9404
9405/**
9406 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9407 */
9408HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9409{
9410 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9411 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9412 HMVMX_RETURN_UNEXPECTED_EXIT();
9413}
9414
9415
9416/**
9417 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9418 */
9419HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9420{
9421 /*
9422 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9423 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9424 * See Intel spec. 25.3 "Other Causes of VM-exits".
9425 */
9426 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9427 HMVMX_RETURN_UNEXPECTED_EXIT();
9428}
9429
9430
9431/**
9432 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9433 * VM-exit.
9434 */
9435HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9436{
9437 /*
9438 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9439 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9440 *
9441 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9442 * See Intel spec. "23.8 Restrictions on VMX operation".
9443 */
9444 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9445 return VINF_SUCCESS;
9446}
9447
9448
9449/**
9450 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9451 * VM-exit.
9452 */
9453HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9454{
9455 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9456 return VINF_EM_RESET;
9457}
9458
9459
9460/**
9461 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9462 */
9463HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9464{
9465 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9466 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9467 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9468 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9469 AssertRCReturn(rc, rc);
9470
9471 pMixedCtx->rip++;
9472 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9473 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9474 rc = VINF_SUCCESS;
9475 else
9476 rc = VINF_EM_HALT;
9477
9478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9479 return rc;
9480}
9481
9482
9483/**
9484 * VM-exit handler for instructions that result in a #UD exception delivered to
9485 * the guest.
9486 */
9487HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9488{
9489 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9490 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9491 return VINF_SUCCESS;
9492}
9493
9494
9495/**
9496 * VM-exit handler for expiry of the VMX preemption timer.
9497 */
9498HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9499{
9500 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9501
9502 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9503 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9504
9505 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9506 PVM pVM = pVCpu->CTX_SUFF(pVM);
9507 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9508 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9509 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9510}
9511
9512
9513/**
9514 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9515 */
9516HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9517{
9518 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9519
9520 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9521 /** @todo check if XSETBV is supported by the recompiler. */
9522 return VERR_EM_INTERPRETER;
9523}
9524
9525
9526/**
9527 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9528 */
9529HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9530{
9531 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9532
9533 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9534 /** @todo implement EMInterpretInvpcid() */
9535 return VERR_EM_INTERPRETER;
9536}
9537
9538
9539/**
9540 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9541 * Error VM-exit.
9542 */
9543HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9544{
9545 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9546 AssertRCReturn(rc, rc);
9547
9548 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9549 NOREF(uInvalidReason);
9550
9551#ifdef VBOX_STRICT
9552 uint32_t uIntrState;
9553 HMVMXHCUINTREG uHCReg;
9554 uint64_t u64Val;
9555 uint32_t u32Val;
9556
9557 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9558 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9559 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9560 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9561 AssertRCReturn(rc, rc);
9562
9563 Log4(("uInvalidReason %u\n", uInvalidReason));
9564 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9565 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9566 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9567 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9568
9569 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9570 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9571 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9572 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9573 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9574 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9575 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9576 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9577 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9578 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9579 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9580 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9581#endif
9582
9583 PVM pVM = pVCpu->CTX_SUFF(pVM);
9584 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9585
9586 return VERR_VMX_INVALID_GUEST_STATE;
9587}
9588
9589
9590/**
9591 * VM-exit handler for VM-entry failure due to an MSR-load
9592 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9593 */
9594HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9595{
9596 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9597 HMVMX_RETURN_UNEXPECTED_EXIT();
9598}
9599
9600
9601/**
9602 * VM-exit handler for VM-entry failure due to a machine-check event
9603 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9604 */
9605HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9606{
9607 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9608 HMVMX_RETURN_UNEXPECTED_EXIT();
9609}
9610
9611
9612/**
9613 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9614 * theory.
9615 */
9616HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9617{
9618 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9619 return VERR_VMX_UNDEFINED_EXIT_CODE;
9620}
9621
9622
9623/**
9624 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9625 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9626 * Conditional VM-exit.
9627 */
9628HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9629{
9630 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9631
9632 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9633 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9634 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9635 return VERR_EM_INTERPRETER;
9636 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9637 HMVMX_RETURN_UNEXPECTED_EXIT();
9638}
9639
9640
9641/**
9642 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9643 */
9644HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9645{
9646 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9647
9648 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9649 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9650 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9651 return VERR_EM_INTERPRETER;
9652 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9653 HMVMX_RETURN_UNEXPECTED_EXIT();
9654}
9655
9656
9657/**
9658 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9659 */
9660HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9661{
9662 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9663
9664 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9665 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9666 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9667 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9668 AssertRCReturn(rc, rc);
9669 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9670
9671 PVM pVM = pVCpu->CTX_SUFF(pVM);
9672 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9673 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9674 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9675 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9676
9677 if (RT_LIKELY(rc == VINF_SUCCESS))
9678 {
9679 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9680 Assert(pVmxTransient->cbInstr == 2);
9681 }
9682 return rc;
9683}
9684
9685
9686/**
9687 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9688 */
9689HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9690{
9691 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9692 PVM pVM = pVCpu->CTX_SUFF(pVM);
9693 int rc = VINF_SUCCESS;
9694
9695 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9696 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9697 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9698 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9699 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9700 AssertRCReturn(rc, rc);
9701 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9702
9703 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9704 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9705 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9706
9707 if (RT_LIKELY(rc == VINF_SUCCESS))
9708 {
9709 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9710
9711 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9712 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9713 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9714 {
9715 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9716 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9717 EMInterpretWrmsr() changes it. */
9718 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9719 }
9720 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9721 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9722 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9723 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9724
9725 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9726 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9727 {
9728 switch (pMixedCtx->ecx)
9729 {
9730 case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9731 case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9732 case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9733 case MSR_K8_FS_BASE: /* no break */
9734 case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9735 case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9736 }
9737 }
9738#ifdef VBOX_STRICT
9739 else
9740 {
9741 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9742 switch (pMixedCtx->ecx)
9743 {
9744 case MSR_IA32_SYSENTER_CS:
9745 case MSR_IA32_SYSENTER_EIP:
9746 case MSR_IA32_SYSENTER_ESP:
9747 case MSR_K8_FS_BASE:
9748 case MSR_K8_GS_BASE:
9749 {
9750 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9751 HMVMX_RETURN_UNEXPECTED_EXIT();
9752 }
9753
9754 case MSR_K8_LSTAR:
9755 case MSR_K6_STAR:
9756 case MSR_K8_SF_MASK:
9757 case MSR_K8_TSC_AUX:
9758 case MSR_K8_KERNEL_GS_BASE:
9759 {
9760 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9761 pMixedCtx->ecx));
9762 HMVMX_RETURN_UNEXPECTED_EXIT();
9763 }
9764 }
9765 }
9766#endif /* VBOX_STRICT */
9767 }
9768 return rc;
9769}
9770
9771
9772/**
9773 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9774 */
9775HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9776{
9777 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9778
9779 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9780 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9781 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9782 return VERR_EM_INTERPRETER;
9783 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9784 HMVMX_RETURN_UNEXPECTED_EXIT();
9785}
9786
9787
9788/**
9789 * VM-exit handler for when the TPR value is lowered below the specified
9790 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9791 */
9792HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9793{
9794 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9795 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9796
9797 /*
9798 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9799 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9800 * resume guest execution.
9801 */
9802 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9804 return VINF_SUCCESS;
9805}
9806
9807
9808/**
9809 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9810 * VM-exit.
9811 *
9812 * @retval VINF_SUCCESS when guest execution can continue.
9813 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9814 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9815 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9816 * recompiler.
9817 */
9818HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819{
9820 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9821 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9822 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9823 AssertRCReturn(rc, rc);
9824
9825 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9826 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9827 PVM pVM = pVCpu->CTX_SUFF(pVM);
9828 switch (uAccessType)
9829 {
9830 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9831 {
9832#if 0
9833 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9834 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9835#else
9836 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9837 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9838 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9839#endif
9840 AssertRCReturn(rc, rc);
9841
9842 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9843 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9844 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9845 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9846
9847 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9848 {
9849 case 0: /* CR0 */
9850 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9851 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9852 break;
9853 case 2: /* CR2 */
9854 /* Nothing to do here, CR2 it's not part of the VMCS. */
9855 break;
9856 case 3: /* CR3 */
9857 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9858 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9859 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9860 break;
9861 case 4: /* CR4 */
9862 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9863 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9864 break;
9865 case 8: /* CR8 */
9866 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9867 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9868 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9869 break;
9870 default:
9871 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9872 break;
9873 }
9874
9875 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9876 break;
9877 }
9878
9879 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9880 {
9881 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9882 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9883 AssertRCReturn(rc, rc);
9884 Assert( !pVM->hm.s.fNestedPaging
9885 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9886 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9887
9888 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9889 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9890 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9891
9892 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9893 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9894 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9895 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9896 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9897 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9898 break;
9899 }
9900
9901 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9902 {
9903 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9904 AssertRCReturn(rc, rc);
9905 rc = EMInterpretCLTS(pVM, pVCpu);
9906 AssertRCReturn(rc, rc);
9907 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9908 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9909 Log4(("CRX CLTS write rc=%d\n", rc));
9910 break;
9911 }
9912
9913 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9914 {
9915 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9916 AssertRCReturn(rc, rc);
9917 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9918 if (RT_LIKELY(rc == VINF_SUCCESS))
9919 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9920 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9921 Log4(("CRX LMSW write rc=%d\n", rc));
9922 break;
9923 }
9924
9925 default:
9926 {
9927 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9928 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9929 }
9930 }
9931
9932 /* Validate possible error codes. */
9933 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9934 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9935 if (RT_SUCCESS(rc))
9936 {
9937 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9938 AssertRCReturn(rc2, rc2);
9939 }
9940
9941 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9942 return rc;
9943}
9944
9945
9946/**
9947 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9948 * VM-exit.
9949 */
9950HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9951{
9952 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9953 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9954
9955 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9956 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9957 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9958 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9959 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9960 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9961 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9962 AssertRCReturn(rc2, rc2);
9963
9964 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9965 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9966 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9967 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9968 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9969 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9970 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9971
9972 /* I/O operation lookup arrays. */
9973 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9974 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9975
9976 VBOXSTRICTRC rcStrict;
9977 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9978 const uint32_t cbInstr = pVmxTransient->cbInstr;
9979 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9980 PVM pVM = pVCpu->CTX_SUFF(pVM);
9981 if (fIOString)
9982 {
9983#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9984 /*
9985 * INS/OUTS - I/O String instruction.
9986 *
9987 * Use instruction-information if available, otherwise fall back on
9988 * interpreting the instruction.
9989 */
9990 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9991 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9992 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9993 {
9994 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9995 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9996 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9997 AssertRCReturn(rc2, rc2);
9998 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9999 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10000 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10001 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10002 if (fIOWrite)
10003 {
10004 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10005 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10006 }
10007 else
10008 {
10009 /*
10010 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10011 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10012 * See Intel Instruction spec. for "INS".
10013 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10014 */
10015 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10016 }
10017 }
10018 else
10019 {
10020 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10021 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10022 AssertRCReturn(rc2, rc2);
10023 rcStrict = IEMExecOne(pVCpu);
10024 }
10025 /** @todo IEM needs to be setting these flags somehow. */
10026 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10027 fUpdateRipAlready = true;
10028#else
10029 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10030 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
10031 if (RT_SUCCESS(rcStrict))
10032 {
10033 if (fIOWrite)
10034 {
10035 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10036 (DISCPUMODE)pDis->uAddrMode, cbValue);
10037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10038 }
10039 else
10040 {
10041 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10042 (DISCPUMODE)pDis->uAddrMode, cbValue);
10043 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10044 }
10045 }
10046 else
10047 {
10048 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10049 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10050 }
10051#endif
10052 }
10053 else
10054 {
10055 /*
10056 * IN/OUT - I/O instruction.
10057 */
10058 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10059 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10060 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10061 if (fIOWrite)
10062 {
10063 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10064 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10065 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10066 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10067 }
10068 else
10069 {
10070 uint32_t u32Result = 0;
10071 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10072 if (IOM_SUCCESS(rcStrict))
10073 {
10074 /* Save result of I/O IN instr. in AL/AX/EAX. */
10075 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10076 }
10077 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10078 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10079 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10080 }
10081 }
10082
10083 if (IOM_SUCCESS(rcStrict))
10084 {
10085 if (!fUpdateRipAlready)
10086 {
10087 pMixedCtx->rip += cbInstr;
10088 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10089 }
10090
10091 /* INS & OUTS with REP prefix modify RFLAGS. */
10092 if (fIOString)
10093 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10094
10095 /*
10096 * If any I/O breakpoints are armed, we need to check if one triggered
10097 * and take appropriate action.
10098 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10099 */
10100 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10101 AssertRCReturn(rc2, rc2);
10102
10103 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10104 * execution engines about whether hyper BPs and such are pending. */
10105 uint32_t const uDr7 = pMixedCtx->dr[7];
10106 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10107 && X86_DR7_ANY_RW_IO(uDr7)
10108 && (pMixedCtx->cr4 & X86_CR4_DE))
10109 || DBGFBpIsHwIoArmed(pVM)))
10110 {
10111 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10112
10113 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10114 VMMRZCallRing3Disable(pVCpu);
10115 HM_DISABLE_PREEMPT_IF_NEEDED();
10116
10117 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10118
10119 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10120 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10121 {
10122 /* Raise #DB. */
10123 if (fIsGuestDbgActive)
10124 ASMSetDR6(pMixedCtx->dr[6]);
10125 if (pMixedCtx->dr[7] != uDr7)
10126 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10127
10128 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10129 }
10130 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10131 else if ( rcStrict2 != VINF_SUCCESS
10132 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10133 rcStrict = rcStrict2;
10134
10135 HM_RESTORE_PREEMPT_IF_NEEDED();
10136 VMMRZCallRing3Enable(pVCpu);
10137 }
10138 }
10139
10140#ifdef DEBUG
10141 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10142 Assert(!fIOWrite);
10143 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10144 Assert(fIOWrite);
10145 else
10146 {
10147 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10148 * statuses, that the VMM device and some others may return. See
10149 * IOM_SUCCESS() for guidance. */
10150 AssertMsg( RT_FAILURE(rcStrict)
10151 || rcStrict == VINF_SUCCESS
10152 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10153 || rcStrict == VINF_EM_DBG_BREAKPOINT
10154 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10155 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10156 }
10157#endif
10158
10159 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10160 return VBOXSTRICTRC_TODO(rcStrict);
10161}
10162
10163
10164/**
10165 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10166 * VM-exit.
10167 */
10168HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10169{
10170 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10171
10172 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10173 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10174 AssertRCReturn(rc, rc);
10175 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10176 {
10177 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10178 AssertRCReturn(rc, rc);
10179 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10180 {
10181 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10182
10183 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10184 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10185 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10186 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10187 {
10188 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10189 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10190
10191 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10192 Assert(!pVCpu->hm.s.Event.fPending);
10193 pVCpu->hm.s.Event.fPending = true;
10194 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10195 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10196 AssertRCReturn(rc, rc);
10197 if (fErrorCodeValid)
10198 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10199 else
10200 pVCpu->hm.s.Event.u32ErrCode = 0;
10201 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10202 && uVector == X86_XCPT_PF)
10203 {
10204 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10205 }
10206
10207 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10208 }
10209 }
10210 }
10211
10212 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10213 * emulation. */
10214 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10215 return VERR_EM_INTERPRETER;
10216}
10217
10218
10219/**
10220 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10221 */
10222HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10223{
10224 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10225 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10226 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10227 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10228 AssertRCReturn(rc, rc);
10229 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10230 return VINF_EM_DBG_STEPPED;
10231}
10232
10233
10234/**
10235 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10236 */
10237HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10238{
10239 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10240
10241 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10242 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10243 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10244 return VINF_SUCCESS;
10245 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10246 return rc;
10247
10248#if 0
10249 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10250 * just sync the whole thing. */
10251 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10252#else
10253 /* Aggressive state sync. for now. */
10254 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10255 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10256 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10257#endif
10258 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10259 AssertRCReturn(rc, rc);
10260
10261 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10262 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10263 switch (uAccessType)
10264 {
10265 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10266 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10267 {
10268 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10269 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10270 {
10271 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10272 }
10273
10274 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10275 GCPhys &= PAGE_BASE_GC_MASK;
10276 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10277 PVM pVM = pVCpu->CTX_SUFF(pVM);
10278 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10279 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10280
10281 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10282 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10283 CPUMCTX2CORE(pMixedCtx), GCPhys);
10284 rc = VBOXSTRICTRC_VAL(rc2);
10285 Log4(("ApicAccess rc=%d\n", rc));
10286 if ( rc == VINF_SUCCESS
10287 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10288 || rc == VERR_PAGE_NOT_PRESENT)
10289 {
10290 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10291 | HM_CHANGED_GUEST_RSP
10292 | HM_CHANGED_GUEST_RFLAGS
10293 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10294 rc = VINF_SUCCESS;
10295 }
10296 break;
10297 }
10298
10299 default:
10300 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10301 rc = VINF_EM_RAW_EMULATE_INSTR;
10302 break;
10303 }
10304
10305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10306 return rc;
10307}
10308
10309
10310/**
10311 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10312 * VM-exit.
10313 */
10314HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10315{
10316 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10317
10318 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10319 if (pVmxTransient->fWasGuestDebugStateActive)
10320 {
10321 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10322 HMVMX_RETURN_UNEXPECTED_EXIT();
10323 }
10324
10325 int rc = VERR_INTERNAL_ERROR_5;
10326 if ( !DBGFIsStepping(pVCpu)
10327 && !pVCpu->hm.s.fSingleInstruction
10328 && !pVmxTransient->fWasHyperDebugStateActive)
10329 {
10330 /* Don't intercept MOV DRx and #DB any more. */
10331 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10332 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10333 AssertRCReturn(rc, rc);
10334
10335 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10336 {
10337#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10338 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10339 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10340 AssertRCReturn(rc, rc);
10341#endif
10342 }
10343
10344 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10345 VMMRZCallRing3Disable(pVCpu);
10346 HM_DISABLE_PREEMPT_IF_NEEDED();
10347
10348 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10349 PVM pVM = pVCpu->CTX_SUFF(pVM);
10350 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10351 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10352
10353 HM_RESTORE_PREEMPT_IF_NEEDED();
10354 VMMRZCallRing3Enable(pVCpu);
10355
10356#ifdef VBOX_WITH_STATISTICS
10357 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10358 AssertRCReturn(rc, rc);
10359 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10361 else
10362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10363#endif
10364 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10365 return VINF_SUCCESS;
10366 }
10367
10368 /*
10369 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10370 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10371 */
10372 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10373 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10374 AssertRCReturn(rc, rc);
10375 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10376
10377 PVM pVM = pVCpu->CTX_SUFF(pVM);
10378 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10379 {
10380 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10381 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10382 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10383 if (RT_SUCCESS(rc))
10384 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10385 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10386 }
10387 else
10388 {
10389 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10390 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10391 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10392 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10393 }
10394
10395 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10396 if (RT_SUCCESS(rc))
10397 {
10398 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10399 AssertRCReturn(rc2, rc2);
10400 }
10401 return rc;
10402}
10403
10404
10405/**
10406 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10407 * Conditional VM-exit.
10408 */
10409HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10410{
10411 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10412 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10413
10414 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10415 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10416 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10417 return VINF_SUCCESS;
10418 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10419 return rc;
10420
10421 RTGCPHYS GCPhys = 0;
10422 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10423
10424#if 0
10425 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10426#else
10427 /* Aggressive state sync. for now. */
10428 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10429 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10430 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10431#endif
10432 AssertRCReturn(rc, rc);
10433
10434 /*
10435 * If we succeed, resume guest execution.
10436 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10437 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10438 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10439 * weird case. See @bugref{6043}.
10440 */
10441 PVM pVM = pVCpu->CTX_SUFF(pVM);
10442 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10443 rc = VBOXSTRICTRC_VAL(rc2);
10444 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10445 if ( rc == VINF_SUCCESS
10446 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10447 || rc == VERR_PAGE_NOT_PRESENT)
10448 {
10449 /* Successfully handled MMIO operation. */
10450 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10451 | HM_CHANGED_GUEST_RSP
10452 | HM_CHANGED_GUEST_RFLAGS
10453 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10454 rc = VINF_SUCCESS;
10455 }
10456 return rc;
10457}
10458
10459
10460/**
10461 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10462 * VM-exit.
10463 */
10464HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10465{
10466 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10467 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10468
10469 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10470 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10471 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10472 return VINF_SUCCESS;
10473 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10474 return rc;
10475
10476 RTGCPHYS GCPhys = 0;
10477 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10478 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10479#if 0
10480 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10481#else
10482 /* Aggressive state sync. for now. */
10483 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10484 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10485 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10486#endif
10487 AssertRCReturn(rc, rc);
10488
10489 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10490 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10491
10492 RTGCUINT uErrorCode = 0;
10493 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10494 uErrorCode |= X86_TRAP_PF_ID;
10495 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10496 uErrorCode |= X86_TRAP_PF_RW;
10497 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10498 uErrorCode |= X86_TRAP_PF_P;
10499
10500 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10501
10502 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10503 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10504
10505 /* Handle the pagefault trap for the nested shadow table. */
10506 PVM pVM = pVCpu->CTX_SUFF(pVM);
10507 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10508 TRPMResetTrap(pVCpu);
10509
10510 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10511 if ( rc == VINF_SUCCESS
10512 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10513 || rc == VERR_PAGE_NOT_PRESENT)
10514 {
10515 /* Successfully synced our nested page tables. */
10516 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10517 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10518 | HM_CHANGED_GUEST_RSP
10519 | HM_CHANGED_GUEST_RFLAGS);
10520 return VINF_SUCCESS;
10521 }
10522
10523 Log4(("EPT return to ring-3 rc=%d\n"));
10524 return rc;
10525}
10526
10527/** @} */
10528
10529/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10530/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10531/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10532
10533/** @name VM-exit exception handlers.
10534 * @{
10535 */
10536
10537/**
10538 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10539 */
10540static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10541{
10542 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10543 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10544
10545 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10546 AssertRCReturn(rc, rc);
10547
10548 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10549 {
10550 /* Old-style FPU error reporting needs some extra work. */
10551 /** @todo don't fall back to the recompiler, but do it manually. */
10552 return VERR_EM_INTERPRETER;
10553 }
10554
10555 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10556 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10557 return rc;
10558}
10559
10560
10561/**
10562 * VM-exit exception handler for #BP (Breakpoint exception).
10563 */
10564static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10565{
10566 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10568
10569 /** @todo Try optimize this by not saving the entire guest state unless
10570 * really needed. */
10571 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10572 AssertRCReturn(rc, rc);
10573
10574 PVM pVM = pVCpu->CTX_SUFF(pVM);
10575 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10576 if (rc == VINF_EM_RAW_GUEST_TRAP)
10577 {
10578 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10579 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10580 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10581 AssertRCReturn(rc, rc);
10582
10583 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10584 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10585 }
10586
10587 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10588 return rc;
10589}
10590
10591
10592/**
10593 * VM-exit exception handler for #DB (Debug exception).
10594 */
10595static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10596{
10597 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10598 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10599 Log6(("XcptDB\n"));
10600
10601 /*
10602 * Get the DR6-like values from the exit qualification and pass it to DBGF
10603 * for processing.
10604 */
10605 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10606 AssertRCReturn(rc, rc);
10607
10608 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10609 uint64_t uDR6 = X86_DR6_INIT_VAL;
10610 uDR6 |= ( pVmxTransient->uExitQualification
10611 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10612
10613 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10614 if (rc == VINF_EM_RAW_GUEST_TRAP)
10615 {
10616 /*
10617 * The exception was for the guest. Update DR6, DR7.GD and
10618 * IA32_DEBUGCTL.LBR before forwarding it.
10619 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10620 */
10621 VMMRZCallRing3Disable(pVCpu);
10622 HM_DISABLE_PREEMPT_IF_NEEDED();
10623
10624 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10625 pMixedCtx->dr[6] |= uDR6;
10626 if (CPUMIsGuestDebugStateActive(pVCpu))
10627 ASMSetDR6(pMixedCtx->dr[6]);
10628
10629 HM_RESTORE_PREEMPT_IF_NEEDED();
10630 VMMRZCallRing3Enable(pVCpu);
10631
10632 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10633 AssertRCReturn(rc, rc);
10634
10635 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10636 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10637
10638 /* Paranoia. */
10639 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10640 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10641
10642 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10643 AssertRCReturn(rc, rc);
10644
10645 /*
10646 * Raise #DB in the guest.
10647 */
10648 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10649 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10650 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10651 AssertRCReturn(rc, rc);
10652 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10653 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10654 return VINF_SUCCESS;
10655 }
10656
10657 /*
10658 * Not a guest trap, must be a hypervisor related debug event then.
10659 * Update DR6 in case someone is interested in it.
10660 */
10661 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10662 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10663 CPUMSetHyperDR6(pVCpu, uDR6);
10664
10665 return rc;
10666}
10667
10668
10669/**
10670 * VM-exit exception handler for #NM (Device-not-available exception: floating
10671 * point exception).
10672 */
10673static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10674{
10675 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10676
10677 /* We require CR0 and EFER. EFER is always up-to-date. */
10678 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10679 AssertRCReturn(rc, rc);
10680
10681 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10682 VMMRZCallRing3Disable(pVCpu);
10683 HM_DISABLE_PREEMPT_IF_NEEDED();
10684
10685 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10686 if (pVmxTransient->fWasGuestFPUStateActive)
10687 {
10688 rc = VINF_EM_RAW_GUEST_TRAP;
10689 Assert(CPUMIsGuestFPUStateActive(pVCpu) || VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10690 }
10691 else
10692 {
10693#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10694 Assert(!pVmxTransient->fWasGuestFPUStateActive);
10695#endif
10696 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10697 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10698 }
10699
10700 HM_RESTORE_PREEMPT_IF_NEEDED();
10701 VMMRZCallRing3Enable(pVCpu);
10702
10703 if (rc == VINF_SUCCESS)
10704 {
10705 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10706 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10707 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10708 }
10709 else
10710 {
10711 /* Forward #NM to the guest. */
10712 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10713 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10714 AssertRCReturn(rc, rc);
10715 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10716 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10718 }
10719
10720 return VINF_SUCCESS;
10721}
10722
10723
10724/**
10725 * VM-exit exception handler for #GP (General-protection exception).
10726 *
10727 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10728 */
10729static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10730{
10731 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10732 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10733
10734 int rc = VERR_INTERNAL_ERROR_5;
10735 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10736 {
10737#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10738 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10739 rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10740 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10741 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10742 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10743 AssertRCReturn(rc, rc);
10744 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10745 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10746 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10747 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10748 return rc;
10749#else
10750 /* We don't intercept #GP. */
10751 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10752 return VERR_VMX_UNEXPECTED_EXCEPTION;
10753#endif
10754 }
10755
10756 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10757 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10758
10759 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10760 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10761 AssertRCReturn(rc, rc);
10762
10763 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10764 uint32_t cbOp = 0;
10765 PVM pVM = pVCpu->CTX_SUFF(pVM);
10766 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10767 if (RT_SUCCESS(rc))
10768 {
10769 rc = VINF_SUCCESS;
10770 Assert(cbOp == pDis->cbInstr);
10771 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10772 switch (pDis->pCurInstr->uOpcode)
10773 {
10774 case OP_CLI:
10775 {
10776 pMixedCtx->eflags.Bits.u1IF = 0;
10777 pMixedCtx->rip += pDis->cbInstr;
10778 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10780 break;
10781 }
10782
10783 case OP_STI:
10784 {
10785 pMixedCtx->eflags.Bits.u1IF = 1;
10786 pMixedCtx->rip += pDis->cbInstr;
10787 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10788 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10789 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10790 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10791 break;
10792 }
10793
10794 case OP_HLT:
10795 {
10796 rc = VINF_EM_HALT;
10797 pMixedCtx->rip += pDis->cbInstr;
10798 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10800 break;
10801 }
10802
10803 case OP_POPF:
10804 {
10805 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10806 uint32_t cbParm = 0;
10807 uint32_t uMask = 0;
10808 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10809 {
10810 cbParm = 4;
10811 uMask = 0xffffffff;
10812 }
10813 else
10814 {
10815 cbParm = 2;
10816 uMask = 0xffff;
10817 }
10818
10819 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10820 RTGCPTR GCPtrStack = 0;
10821 X86EFLAGS Eflags;
10822 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10823 &GCPtrStack);
10824 if (RT_SUCCESS(rc))
10825 {
10826 Assert(sizeof(Eflags.u32) >= cbParm);
10827 Eflags.u32 = 0;
10828 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10829 }
10830 if (RT_FAILURE(rc))
10831 {
10832 rc = VERR_EM_INTERPRETER;
10833 break;
10834 }
10835 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10836 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10837 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10838 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10839 pMixedCtx->esp += cbParm;
10840 pMixedCtx->esp &= uMask;
10841 pMixedCtx->rip += pDis->cbInstr;
10842
10843 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10844 | HM_CHANGED_GUEST_RSP
10845 | HM_CHANGED_GUEST_RFLAGS);
10846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10847 break;
10848 }
10849
10850 case OP_PUSHF:
10851 {
10852 uint32_t cbParm = 0;
10853 uint32_t uMask = 0;
10854 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10855 {
10856 cbParm = 4;
10857 uMask = 0xffffffff;
10858 }
10859 else
10860 {
10861 cbParm = 2;
10862 uMask = 0xffff;
10863 }
10864
10865 /* Get the stack pointer & push the contents of eflags onto the stack. */
10866 RTGCPTR GCPtrStack = 0;
10867 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10868 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10869 if (RT_FAILURE(rc))
10870 {
10871 rc = VERR_EM_INTERPRETER;
10872 break;
10873 }
10874 X86EFLAGS Eflags = pMixedCtx->eflags;
10875 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10876 Eflags.Bits.u1RF = 0;
10877 Eflags.Bits.u1VM = 0;
10878
10879 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10880 if (RT_FAILURE(rc))
10881 {
10882 rc = VERR_EM_INTERPRETER;
10883 break;
10884 }
10885 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10886 pMixedCtx->esp -= cbParm;
10887 pMixedCtx->esp &= uMask;
10888 pMixedCtx->rip += pDis->cbInstr;
10889 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
10890 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10891 break;
10892 }
10893
10894 case OP_IRET:
10895 {
10896 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10897 * instruction reference. */
10898 RTGCPTR GCPtrStack = 0;
10899 uint32_t uMask = 0xffff;
10900 uint16_t aIretFrame[3];
10901 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10902 {
10903 rc = VERR_EM_INTERPRETER;
10904 break;
10905 }
10906 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10907 &GCPtrStack);
10908 if (RT_SUCCESS(rc))
10909 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10910 if (RT_FAILURE(rc))
10911 {
10912 rc = VERR_EM_INTERPRETER;
10913 break;
10914 }
10915 pMixedCtx->eip = 0;
10916 pMixedCtx->ip = aIretFrame[0];
10917 pMixedCtx->cs.Sel = aIretFrame[1];
10918 pMixedCtx->cs.ValidSel = aIretFrame[1];
10919 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10920 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10921 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10922 pMixedCtx->sp += sizeof(aIretFrame);
10923 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10924 | HM_CHANGED_GUEST_SEGMENT_REGS
10925 | HM_CHANGED_GUEST_RSP
10926 | HM_CHANGED_GUEST_RFLAGS);
10927 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10928 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10929 break;
10930 }
10931
10932 case OP_INT:
10933 {
10934 uint16_t uVector = pDis->Param1.uValue & 0xff;
10935 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10936 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10937 break;
10938 }
10939
10940 case OP_INTO:
10941 {
10942 if (pMixedCtx->eflags.Bits.u1OF)
10943 {
10944 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10946 }
10947 break;
10948 }
10949
10950 default:
10951 {
10952 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10953 EMCODETYPE_SUPERVISOR);
10954 rc = VBOXSTRICTRC_VAL(rc2);
10955 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10956 Log4(("#GP rc=%Rrc\n", rc));
10957 break;
10958 }
10959 }
10960 }
10961 else
10962 rc = VERR_EM_INTERPRETER;
10963
10964 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10965 ("#GP Unexpected rc=%Rrc\n", rc));
10966 return rc;
10967}
10968
10969
10970/**
10971 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10972 * the exception reported in the VMX transient structure back into the VM.
10973 *
10974 * @remarks Requires uExitIntInfo in the VMX transient structure to be
10975 * up-to-date.
10976 */
10977static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10978{
10979 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10980
10981 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10982 hmR0VmxCheckExitDueToEventDelivery(). */
10983 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10984 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10985 AssertRCReturn(rc, rc);
10986 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10987
10988 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10989 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10990 return VINF_SUCCESS;
10991}
10992
10993
10994/**
10995 * VM-exit exception handler for #PF (Page-fault exception).
10996 */
10997static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10998{
10999 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11000 PVM pVM = pVCpu->CTX_SUFF(pVM);
11001 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11002 rc |= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
11003 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11004 AssertRCReturn(rc, rc);
11005
11006#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11007 if (pVM->hm.s.fNestedPaging)
11008 {
11009 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11010 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11011 {
11012 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11013 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11014 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11015 }
11016 else
11017 {
11018 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11019 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11020 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11021 }
11022 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11023 return rc;
11024 }
11025#else
11026 Assert(!pVM->hm.s.fNestedPaging);
11027#endif
11028
11029 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11030 AssertRCReturn(rc, rc);
11031
11032 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11033 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11034
11035 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11036 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11037 (RTGCPTR)pVmxTransient->uExitQualification);
11038
11039 Log4(("#PF: rc=%Rrc\n", rc));
11040 if (rc == VINF_SUCCESS)
11041 {
11042 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11043 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11044 * memory? We don't update the whole state here... */
11045 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11046 | HM_CHANGED_GUEST_RSP
11047 | HM_CHANGED_GUEST_RFLAGS
11048 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11049 TRPMResetTrap(pVCpu);
11050 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11051 return rc;
11052 }
11053 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11054 {
11055 if (!pVmxTransient->fVectoringPF)
11056 {
11057 /* It's a guest page fault and needs to be reflected to the guest. */
11058 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11059 TRPMResetTrap(pVCpu);
11060 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11061 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11062 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11063 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11064 }
11065 else
11066 {
11067 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11068 TRPMResetTrap(pVCpu);
11069 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11070 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11071 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11072 }
11073
11074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11075 return VINF_SUCCESS;
11076 }
11077
11078 TRPMResetTrap(pVCpu);
11079 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11080 return rc;
11081}
11082
11083/** @} */
11084
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette