VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 51182

Last change on this file since 51182 was 51182, checked in by vboxsync, 11 years ago

VMM/IEM: Implemented hardware task-switches, code path disabled.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 479.4 KB
Line 
1/* $Id: HMVMXR0.cpp 51182 2014-05-05 12:08:40Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_SAVE_FULL_GUEST_STATE
41# define HMVMX_SYNC_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44# define HMVMX_ALWAYS_TRAP_PF
45# define HMVMX_ALWAYS_SWAP_FPU_STATE
46# define HMVMX_ALWAYS_FLUSH_TLB
47#endif
48
49
50/*******************************************************************************
51* Defined Constants And Macros *
52*******************************************************************************/
53#if defined(RT_ARCH_AMD64)
54# define HMVMX_IS_64BIT_HOST_MODE() (true)
55typedef RTHCUINTREG HMVMXHCUINTREG;
56#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57extern "C" uint32_t g_fVMXIs64bitHost;
58# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59typedef uint64_t HMVMXHCUINTREG;
60#else
61# define HMVMX_IS_64BIT_HOST_MODE() (false)
62typedef RTHCUINTREG HMVMXHCUINTREG;
63#endif
64
65/** Use the function table. */
66#define HMVMX_USE_FUNCTION_TABLE
67
68/** Determine which tagged-TLB flush handler to use. */
69#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74/** @name Updated-guest-state flags.
75 * @{ */
76#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96 | HMVMX_UPDATED_GUEST_RSP \
97 | HMVMX_UPDATED_GUEST_RFLAGS \
98 | HMVMX_UPDATED_GUEST_CR0 \
99 | HMVMX_UPDATED_GUEST_CR3 \
100 | HMVMX_UPDATED_GUEST_CR4 \
101 | HMVMX_UPDATED_GUEST_GDTR \
102 | HMVMX_UPDATED_GUEST_IDTR \
103 | HMVMX_UPDATED_GUEST_LDTR \
104 | HMVMX_UPDATED_GUEST_TR \
105 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106 | HMVMX_UPDATED_GUEST_DEBUG \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
112 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113 | HMVMX_UPDATED_GUEST_APIC_STATE)
114/** @} */
115
116/** @name
117 * Flags to skip redundant reads of some common VMCS fields that are not part of
118 * the guest-CPU state but are in the transient structure.
119 */
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127/** @} */
128
129/** @name
130 * States of the VMCS.
131 *
132 * This does not reflect all possible VMCS states but currently only those
133 * needed for maintaining the VMCS consistently even when thread-context hooks
134 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
135 */
136#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139/** @} */
140
141/**
142 * Exception bitmap mask for real-mode guests (real-on-v86).
143 *
144 * We need to intercept all exceptions manually (except #PF). #NM is also
145 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146 * even in real-mode if we have Nested Paging support.
147 */
148#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
149 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
150 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
151 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
152 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
154 | RT_BIT(X86_XCPT_XF))
155
156/**
157 * Exception bitmap mask for all contributory exceptions.
158 *
159 * Page fault is deliberately excluded here as it's conditional as to whether
160 * it's contributory or benign. Page faults are handled separately.
161 */
162#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
163 | RT_BIT(X86_XCPT_DE))
164
165/** Maximum VM-instruction error number. */
166#define HMVMX_INSTR_ERROR_MAX 28
167
168/** Profiling macro. */
169#ifdef HM_PROFILE_EXIT_DISPATCH
170# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172#else
173# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175#endif
176
177/** Assert that preemption is disabled or covered by thread-context hooks. */
178#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181/** Assert that we haven't migrated CPUs when thread-context hooks are not
182 * used. */
183#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185 ("Illegal migration! Entered on CPU %u Current %u\n", \
186 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188/** Helper macro for VM-exit handlers called unexpectedly. */
189#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190 do { \
191 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192 return VERR_VMX_UNEXPECTED_EXIT; \
193 } while (0)
194
195
196/*******************************************************************************
197* Structures and Typedefs *
198*******************************************************************************/
199/**
200 * VMX transient state.
201 *
202 * A state structure for holding miscellaneous information across
203 * VMX non-root operation and restored after the transition.
204 */
205typedef struct VMXTRANSIENT
206{
207 /** The host's rflags/eflags. */
208 RTCCUINTREG uEflags;
209#if HC_ARCH_BITS == 32
210 uint32_t u32Alignment0;
211#endif
212 /** The guest's TPR value used for TPR shadowing. */
213 uint8_t u8GuestTpr;
214 /** Alignment. */
215 uint8_t abAlignment0[7];
216
217 /** The basic VM-exit reason. */
218 uint16_t uExitReason;
219 /** Alignment. */
220 uint16_t u16Alignment0;
221 /** The VM-exit interruption error code. */
222 uint32_t uExitIntErrorCode;
223 /** The VM-exit exit qualification. */
224 uint64_t uExitQualification;
225
226 /** The VM-exit interruption-information field. */
227 uint32_t uExitIntInfo;
228 /** The VM-exit instruction-length field. */
229 uint32_t cbInstr;
230 /** The VM-exit instruction-information field. */
231 union
232 {
233 /** Plain unsigned int representation. */
234 uint32_t u;
235 /** INS and OUTS information. */
236 struct
237 {
238 uint32_t u6Reserved0 : 7;
239 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240 uint32_t u3AddrSize : 3;
241 uint32_t u5Reserved1 : 5;
242 /** The segment register (X86_SREG_XXX). */
243 uint32_t iSegReg : 3;
244 uint32_t uReserved2 : 14;
245 } StrIo;
246 } ExitInstrInfo;
247 /** Whether the VM-entry failed or not. */
248 bool fVMEntryFailed;
249 /** Alignment. */
250 uint8_t abAlignment1[3];
251
252 /** The VM-entry interruption-information field. */
253 uint32_t uEntryIntInfo;
254 /** The VM-entry exception error code field. */
255 uint32_t uEntryXcptErrorCode;
256 /** The VM-entry instruction length field. */
257 uint32_t cbEntryInstr;
258
259 /** IDT-vectoring information field. */
260 uint32_t uIdtVectoringInfo;
261 /** IDT-vectoring error code. */
262 uint32_t uIdtVectoringErrorCode;
263
264 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
265 uint32_t fVmcsFieldsRead;
266
267 /** Whether the guest FPU was active at the time of VM-exit. */
268 bool fWasGuestFPUStateActive;
269 /** Whether the guest debug state was active at the time of VM-exit. */
270 bool fWasGuestDebugStateActive;
271 /** Whether the hyper debug state was active at the time of VM-exit. */
272 bool fWasHyperDebugStateActive;
273 /** Whether TSC-offsetting should be setup before VM-entry. */
274 bool fUpdateTscOffsettingAndPreemptTimer;
275 /** Whether the VM-exit was caused by a page-fault during delivery of a
276 * contributory exception or a page-fault. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns VBox status code.
319 * @param pVCpu Pointer to the VMCPU.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333
334/*******************************************************************************
335* Internal Functions *
336*******************************************************************************/
337static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343#endif
344#ifndef HMVMX_USE_FUNCTION_TABLE
345DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346# define HMVMX_EXIT_DECL static int
347#else
348# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349#endif
350
351/** @name VM-exit handlers.
352 * @{
353 */
354static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358static FNVMXEXITHANDLER hmR0VmxExitSipi;
359static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360static FNVMXEXITHANDLER hmR0VmxExitSmi;
361static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366static FNVMXEXITHANDLER hmR0VmxExitHlt;
367static FNVMXEXITHANDLER hmR0VmxExitInvd;
368static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371static FNVMXEXITHANDLER hmR0VmxExitRsm;
372static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381static FNVMXEXITHANDLER hmR0VmxExitMwait;
382static FNVMXEXITHANDLER hmR0VmxExitMtf;
383static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384static FNVMXEXITHANDLER hmR0VmxExitPause;
385static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398/** @} */
399
400static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408#endif
409static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411/*******************************************************************************
412* Global Variables *
413*******************************************************************************/
414#ifdef HMVMX_USE_FUNCTION_TABLE
415
416/**
417 * VMX_EXIT dispatch table.
418 */
419static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420{
421 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461 /* 40 UNDEFINED */ hmR0VmxExitPause,
462 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481};
482#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484#ifdef VBOX_STRICT
485static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486{
487 /* 0 */ "(Not Used)",
488 /* 1 */ "VMCALL executed in VMX root operation.",
489 /* 2 */ "VMCLEAR with invalid physical address.",
490 /* 3 */ "VMCLEAR with VMXON pointer.",
491 /* 4 */ "VMLAUNCH with non-clear VMCS.",
492 /* 5 */ "VMRESUME with non-launched VMCS.",
493 /* 6 */ "VMRESUME after VMXOFF",
494 /* 7 */ "VM entry with invalid control fields.",
495 /* 8 */ "VM entry with invalid host state fields.",
496 /* 9 */ "VMPTRLD with invalid physical address.",
497 /* 10 */ "VMPTRLD with VMXON pointer.",
498 /* 11 */ "VMPTRLD with incorrect revision identifier.",
499 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500 /* 13 */ "VMWRITE to read-only VMCS component.",
501 /* 14 */ "(Not Used)",
502 /* 15 */ "VMXON executed in VMX root operation.",
503 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
504 /* 17 */ "VM entry with non-launched executing VMCS.",
505 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506 /* 19 */ "VMCALL with non-clear VMCS.",
507 /* 20 */ "VMCALL with invalid VM-exit control fields.",
508 /* 21 */ "(Not Used)",
509 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511 /* 24 */ "VMCALL with invalid SMM-monitor features.",
512 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513 /* 26 */ "VM entry with events blocked by MOV SS.",
514 /* 27 */ "(Not Used)",
515 /* 28 */ "Invalid operand to INVEPT/INVVPID."
516};
517#endif /* VBOX_STRICT */
518
519
520
521/**
522 * Updates the VM's last error record. If there was a VMX instruction error,
523 * reads the error data from the VMCS and updates VCPU's last error record as
524 * well.
525 *
526 * @param pVM Pointer to the VM.
527 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528 * VERR_VMX_UNABLE_TO_START_VM or
529 * VERR_VMX_INVALID_VMCS_FIELD).
530 * @param rc The error code.
531 */
532static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533{
534 AssertPtr(pVM);
535 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536 || rc == VERR_VMX_UNABLE_TO_START_VM)
537 {
538 AssertPtrReturnVoid(pVCpu);
539 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540 }
541 pVM->hm.s.lLastError = rc;
542}
543
544
545/**
546 * Reads the VM-entry interruption-information field from the VMCS into the VMX
547 * transient structure.
548 *
549 * @returns VBox status code.
550 * @param pVmxTransient Pointer to the VMX transient structure.
551 *
552 * @remarks No-long-jump zone!!!
553 */
554DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555{
556 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557 AssertRCReturn(rc, rc);
558 return VINF_SUCCESS;
559}
560
561
562/**
563 * Reads the VM-entry exception error code field from the VMCS into
564 * the VMX transient structure.
565 *
566 * @returns VBox status code.
567 * @param pVmxTransient Pointer to the VMX transient structure.
568 *
569 * @remarks No-long-jump zone!!!
570 */
571DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572{
573 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574 AssertRCReturn(rc, rc);
575 return VINF_SUCCESS;
576}
577
578
579/**
580 * Reads the VM-entry exception error code field from the VMCS into
581 * the VMX transient structure.
582 *
583 * @returns VBox status code.
584 * @param pVmxTransient Pointer to the VMX transient structure.
585 *
586 * @remarks No-long-jump zone!!!
587 */
588DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589{
590 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591 AssertRCReturn(rc, rc);
592 return VINF_SUCCESS;
593}
594
595
596/**
597 * Reads the VM-exit interruption-information field from the VMCS into the VMX
598 * transient structure.
599 *
600 * @returns VBox status code.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 */
603DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606 {
607 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608 AssertRCReturn(rc, rc);
609 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610 }
611 return VINF_SUCCESS;
612}
613
614
615/**
616 * Reads the VM-exit interruption error code from the VMCS into the VMX
617 * transient structure.
618 *
619 * @returns VBox status code.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction length field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu Pointer to the VMCPU.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the VM-exit instruction-information field from the VMCS into
656 * the VMX transient structure.
657 *
658 * @returns VBox status code.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664 {
665 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the exit qualification from the VMCS into the VMX transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678 * case).
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684 {
685 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the IDT-vectoring information field from the VMCS into the VMX
695 * transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 *
700 * @remarks No-long-jump zone!!!
701 */
702DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703{
704 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705 {
706 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707 AssertRCReturn(rc, rc);
708 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709 }
710 return VINF_SUCCESS;
711}
712
713
714/**
715 * Reads the IDT-vectoring error code from the VMCS into the VMX
716 * transient structure.
717 *
718 * @returns VBox status code.
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724 {
725 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Enters VMX root mode operation on the current CPU.
735 *
736 * @returns VBox status code.
737 * @param pVM Pointer to the VM (optional, can be NULL, after
738 * a resume).
739 * @param HCPhysCpuPage Physical address of the VMXON region.
740 * @param pvCpuPage Pointer to the VMXON region.
741 */
742static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743{
744 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
745 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
746 Assert(pvCpuPage);
747 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
748
749 if (pVM)
750 {
751 /* Write the VMCS revision dword to the VMXON region. */
752 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
753 }
754
755 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
756 RTCCUINTREG uEflags = ASMIntDisableFlags();
757
758 /* Enable the VMX bit in CR4 if necessary. */
759 RTCCUINTREG uCr4 = ASMGetCR4();
760 if (!(uCr4 & X86_CR4_VMXE))
761 ASMSetCR4(uCr4 | X86_CR4_VMXE);
762
763 /* Enter VMX root mode. */
764 int rc = VMXEnable(HCPhysCpuPage);
765 if (RT_FAILURE(rc))
766 ASMSetCR4(uCr4);
767
768 /* Restore interrupts. */
769 ASMSetFlags(uEflags);
770 return rc;
771}
772
773
774/**
775 * Exits VMX root mode operation on the current CPU.
776 *
777 * @returns VBox status code.
778 */
779static int hmR0VmxLeaveRootMode(void)
780{
781 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
782
783 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
784 RTCCUINTREG uEflags = ASMIntDisableFlags();
785
786 /* If we're for some reason not in VMX root mode, then don't leave it. */
787 RTCCUINTREG uHostCR4 = ASMGetCR4();
788
789 int rc;
790 if (uHostCR4 & X86_CR4_VMXE)
791 {
792 /* Exit VMX root mode and clear the VMX bit in CR4. */
793 VMXDisable();
794 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
795 rc = VINF_SUCCESS;
796 }
797 else
798 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
799
800 /* Restore interrupts. */
801 ASMSetFlags(uEflags);
802 return rc;
803}
804
805
806/**
807 * Allocates and maps one physically contiguous page. The allocated page is
808 * zero'd out. (Used by various VT-x structures).
809 *
810 * @returns IPRT status code.
811 * @param pMemObj Pointer to the ring-0 memory object.
812 * @param ppVirt Where to store the virtual address of the
813 * allocation.
814 * @param pPhys Where to store the physical address of the
815 * allocation.
816 */
817DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
818{
819 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
820 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
821 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
822
823 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
824 if (RT_FAILURE(rc))
825 return rc;
826 *ppVirt = RTR0MemObjAddress(*pMemObj);
827 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
828 ASMMemZero32(*ppVirt, PAGE_SIZE);
829 return VINF_SUCCESS;
830}
831
832
833/**
834 * Frees and unmaps an allocated physical page.
835 *
836 * @param pMemObj Pointer to the ring-0 memory object.
837 * @param ppVirt Where to re-initialize the virtual address of
838 * allocation as 0.
839 * @param pHCPhys Where to re-initialize the physical address of the
840 * allocation as 0.
841 */
842DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
843{
844 AssertPtr(pMemObj);
845 AssertPtr(ppVirt);
846 AssertPtr(pHCPhys);
847 if (*pMemObj != NIL_RTR0MEMOBJ)
848 {
849 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
850 AssertRC(rc);
851 *pMemObj = NIL_RTR0MEMOBJ;
852 *ppVirt = 0;
853 *pHCPhys = 0;
854 }
855}
856
857
858/**
859 * Worker function to free VT-x related structures.
860 *
861 * @returns IPRT status code.
862 * @param pVM Pointer to the VM.
863 */
864static void hmR0VmxStructsFree(PVM pVM)
865{
866 for (VMCPUID i = 0; i < pVM->cCpus; i++)
867 {
868 PVMCPU pVCpu = &pVM->aCpus[i];
869 AssertPtr(pVCpu);
870
871 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
872 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873
874 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879 }
880
881 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884#endif
885}
886
887
888/**
889 * Worker function to allocate VT-x related VM structures.
890 *
891 * @returns IPRT status code.
892 * @param pVM Pointer to the VM.
893 */
894static int hmR0VmxStructsAlloc(PVM pVM)
895{
896 /*
897 * Initialize members up-front so we can cleanup properly on allocation failure.
898 */
899#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902 pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911#endif
912 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915 for (VMCPUID i = 0; i < pVM->cCpus; i++)
916 {
917 PVMCPU pVCpu = &pVM->aCpus[i];
918 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
923 }
924#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
925#undef VMXLOCAL_INIT_VM_MEMOBJ
926
927 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
928 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
929 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
930 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
931
932 /*
933 * Allocate all the VT-x structures.
934 */
935 int rc = VINF_SUCCESS;
936#ifdef VBOX_WITH_CRASHDUMP_MAGIC
937 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
938 if (RT_FAILURE(rc))
939 goto cleanup;
940 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
941 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
942#endif
943
944 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
945 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
946 {
947 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
948 &pVM->hm.s.vmx.HCPhysApicAccess);
949 if (RT_FAILURE(rc))
950 goto cleanup;
951 }
952
953 /*
954 * Initialize per-VCPU VT-x structures.
955 */
956 for (VMCPUID i = 0; i < pVM->cCpus; i++)
957 {
958 PVMCPU pVCpu = &pVM->aCpus[i];
959 AssertPtr(pVCpu);
960
961 /* Allocate the VM control structure (VMCS). */
962 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
963 if (RT_FAILURE(rc))
964 goto cleanup;
965
966 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
967 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
968 {
969 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
970 &pVCpu->hm.s.vmx.HCPhysVirtApic);
971 if (RT_FAILURE(rc))
972 goto cleanup;
973 }
974
975 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
976 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
977 {
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
979 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
983 }
984
985 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
986 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989
990 /* Allocate the VM-exit MSR-load page for the host MSRs. */
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
992 if (RT_FAILURE(rc))
993 goto cleanup;
994 }
995
996 return VINF_SUCCESS;
997
998cleanup:
999 hmR0VmxStructsFree(pVM);
1000 return rc;
1001}
1002
1003
1004/**
1005 * Does global VT-x initialization (called during module initialization).
1006 *
1007 * @returns VBox status code.
1008 */
1009VMMR0DECL(int) VMXR0GlobalInit(void)
1010{
1011#ifdef HMVMX_USE_FUNCTION_TABLE
1012 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1013# ifdef VBOX_STRICT
1014 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1015 Assert(g_apfnVMExitHandlers[i]);
1016# endif
1017#endif
1018 return VINF_SUCCESS;
1019}
1020
1021
1022/**
1023 * Does global VT-x termination (called during module termination).
1024 */
1025VMMR0DECL(void) VMXR0GlobalTerm()
1026{
1027 /* Nothing to do currently. */
1028}
1029
1030
1031/**
1032 * Sets up and activates VT-x on the current CPU.
1033 *
1034 * @returns VBox status code.
1035 * @param pCpu Pointer to the global CPU info struct.
1036 * @param pVM Pointer to the VM (can be NULL after a host resume
1037 * operation).
1038 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1039 * fEnabledByHost is true).
1040 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1041 * @a fEnabledByHost is true).
1042 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1043 * enable VT-x on the host.
1044 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1045 */
1046VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1047 void *pvMsrs)
1048{
1049 Assert(pCpu);
1050 Assert(pvMsrs);
1051 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1052
1053 /* Enable VT-x if it's not already enabled by the host. */
1054 if (!fEnabledByHost)
1055 {
1056 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1057 if (RT_FAILURE(rc))
1058 return rc;
1059 }
1060
1061 /*
1062 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1063 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1064 */
1065 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1066 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1067 {
1068 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1069 pCpu->fFlushAsidBeforeUse = false;
1070 }
1071 else
1072 pCpu->fFlushAsidBeforeUse = true;
1073
1074 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1075 ++pCpu->cTlbFlushes;
1076
1077 return VINF_SUCCESS;
1078}
1079
1080
1081/**
1082 * Deactivates VT-x on the current CPU.
1083 *
1084 * @returns VBox status code.
1085 * @param pCpu Pointer to the global CPU info struct.
1086 * @param pvCpuPage Pointer to the VMXON region.
1087 * @param HCPhysCpuPage Physical address of the VMXON region.
1088 *
1089 * @remarks This function should never be called when SUPR0EnableVTx() or
1090 * similar was used to enable VT-x on the host.
1091 */
1092VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1093{
1094 NOREF(pCpu);
1095 NOREF(pvCpuPage);
1096 NOREF(HCPhysCpuPage);
1097
1098 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1099 return hmR0VmxLeaveRootMode();
1100}
1101
1102
1103/**
1104 * Sets the permission bits for the specified MSR in the MSR bitmap.
1105 *
1106 * @param pVCpu Pointer to the VMCPU.
1107 * @param uMSR The MSR value.
1108 * @param enmRead Whether reading this MSR causes a VM-exit.
1109 * @param enmWrite Whether writing this MSR causes a VM-exit.
1110 */
1111static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1112{
1113 int32_t iBit;
1114 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1115
1116 /*
1117 * Layout:
1118 * 0x000 - 0x3ff - Low MSR read bits
1119 * 0x400 - 0x7ff - High MSR read bits
1120 * 0x800 - 0xbff - Low MSR write bits
1121 * 0xc00 - 0xfff - High MSR write bits
1122 */
1123 if (uMsr <= 0x00001FFF)
1124 iBit = uMsr;
1125 else if ( uMsr >= 0xC0000000
1126 && uMsr <= 0xC0001FFF)
1127 {
1128 iBit = (uMsr - 0xC0000000);
1129 pbMsrBitmap += 0x400;
1130 }
1131 else
1132 {
1133 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1134 return;
1135 }
1136
1137 Assert(iBit <= 0x1fff);
1138 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1139 ASMBitSet(pbMsrBitmap, iBit);
1140 else
1141 ASMBitClear(pbMsrBitmap, iBit);
1142
1143 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1144 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1145 else
1146 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1147}
1148
1149
1150#ifdef VBOX_STRICT
1151/**
1152 * Gets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @returns VBox status code.
1155 * @retval VINF_SUCCESS if the specified MSR is found.
1156 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1157 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1158 *
1159 * @param pVCpu Pointer to the VMCPU.
1160 * @param uMsr The MSR.
1161 * @param penmRead Where to store the read permissions.
1162 * @param penmWrite Where to store the write permissions.
1163 */
1164static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1165{
1166 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1167 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1168 int32_t iBit;
1169 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1170
1171 /* See hmR0VmxSetMsrPermission() for the layout. */
1172 if (uMsr <= 0x00001FFF)
1173 iBit = uMsr;
1174 else if ( uMsr >= 0xC0000000
1175 && uMsr <= 0xC0001FFF)
1176 {
1177 iBit = (uMsr - 0xC0000000);
1178 pbMsrBitmap += 0x400;
1179 }
1180 else
1181 {
1182 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1183 return VERR_NOT_SUPPORTED;
1184 }
1185
1186 Assert(iBit <= 0x1fff);
1187 if (ASMBitTest(pbMsrBitmap, iBit))
1188 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1189 else
1190 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1191
1192 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1193 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1194 else
1195 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1196 return VINF_SUCCESS;
1197}
1198#endif /* VBOX_STRICT */
1199
1200
1201/**
1202 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1203 * area.
1204 *
1205 * @returns VBox status code.
1206 * @param pVCpu Pointer to the VMCPU.
1207 * @param cMsrs The number of MSRs.
1208 */
1209DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1210{
1211 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1212 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1213 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1214 {
1215 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1216 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1218 }
1219
1220 /* Update number of guest MSRs to load/store across the world-switch. */
1221 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223
1224 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1225 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1226
1227 /* Update the VCPU's copy of the MSR count. */
1228 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1229
1230 return VINF_SUCCESS;
1231}
1232
1233
1234/**
1235 * Adds a new (or updates the value of an existing) guest/host MSR
1236 * pair to be swapped during the world-switch as part of the
1237 * auto-load/store MSR area in the VMCS.
1238 *
1239 * @returns true if the MSR was added -and- its value was updated, false
1240 * otherwise.
1241 * @param pVCpu Pointer to the VMCPU.
1242 * @param uMsr The MSR.
1243 * @param uGuestMsr Value of the guest MSR.
1244 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1245 * necessary.
1246 */
1247static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1248{
1249 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1250 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1251 uint32_t i;
1252 for (i = 0; i < cMsrs; i++)
1253 {
1254 if (pGuestMsr->u32Msr == uMsr)
1255 break;
1256 pGuestMsr++;
1257 }
1258
1259 bool fAdded = false;
1260 if (i == cMsrs)
1261 {
1262 ++cMsrs;
1263 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1264 AssertRC(rc);
1265
1266 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1267 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1268 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1269
1270 fAdded = true;
1271 }
1272
1273 /* Update the MSR values in the auto-load/store MSR area. */
1274 pGuestMsr->u32Msr = uMsr;
1275 pGuestMsr->u64Value = uGuestMsrValue;
1276
1277 /* Create/update the MSR slot in the host MSR area. */
1278 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1279 pHostMsr += i;
1280 pHostMsr->u32Msr = uMsr;
1281
1282 /*
1283 * Update the host MSR only when requested by the caller AND when we're
1284 * adding it to the auto-load/store area. Otherwise, it would have been
1285 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1286 */
1287 bool fUpdatedMsrValue = false;
1288 if ( fAdded
1289 && fUpdateHostMsr)
1290 {
1291 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1292 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1293 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1294 fUpdatedMsrValue = true;
1295 }
1296
1297 return fUpdatedMsrValue;
1298}
1299
1300
1301/**
1302 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1303 * auto-load/store MSR area in the VMCS.
1304 *
1305 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1306 * area.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1316 for (uint32_t i = 0; i < cMsrs; i++)
1317 {
1318 /* Find the MSR. */
1319 if (pGuestMsr->u32Msr == uMsr)
1320 {
1321 /* If it's the last MSR, simply reduce the count. */
1322 if (i == cMsrs - 1)
1323 {
1324 --cMsrs;
1325 break;
1326 }
1327
1328 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1329 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1330 pLastGuestMsr += cMsrs;
1331 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1332 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1333
1334 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1336 pLastHostMsr += cMsrs;
1337 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1338 pHostMsr->u64Value = pLastHostMsr->u64Value;
1339 --cMsrs;
1340 break;
1341 }
1342 pGuestMsr++;
1343 }
1344
1345 /* Update the VMCS if the count changed (meaning the MSR was found). */
1346 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1347 {
1348 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1349 AssertRCReturn(rc, rc);
1350
1351 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1352 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1353 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1354 }
1355
1356 return VINF_SUCCESS;
1357}
1358
1359
1360/**
1361 * Checks if the specified guest MSR is part of the auto-load/store area in
1362 * the VMCS.
1363 *
1364 * @returns true if found, false otherwise.
1365 * @param pVCpu Pointer to the VMCPU.
1366 * @param uMsr The MSR to find.
1367 */
1368static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1369{
1370 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1371 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1372
1373 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1374 {
1375 if (pGuestMsr->u32Msr == uMsr)
1376 return true;
1377 }
1378 return false;
1379}
1380
1381
1382/**
1383 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1384 *
1385 * @param pVCpu Pointer to the VMCPU.
1386 *
1387 * @remarks No-long-jump zone!!!
1388 */
1389static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1390{
1391 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1392 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1393 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1394 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1395
1396 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1397 {
1398 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1399 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1400 }
1401
1402 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1403}
1404
1405
1406#if HC_ARCH_BITS == 64
1407/**
1408 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1409 * perform lazy restoration of the host MSRs while leaving VT-x.
1410 *
1411 * @param pVCpu Pointer to the VMCPU.
1412 *
1413 * @remarks No-long-jump zone!!!
1414 */
1415static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1416{
1417 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1418
1419 /*
1420 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1421 */
1422 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1423 {
1424 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1425 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1426 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1427 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1428 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1429 }
1430}
1431
1432
1433/**
1434 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1435 * lazily while leaving VT-x.
1436 *
1437 * @returns true if it does, false otherwise.
1438 * @param pVCpu Pointer to the VMCPU.
1439 * @param uMsr The MSR to check.
1440 */
1441static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1442{
1443 NOREF(pVCpu);
1444 switch (uMsr)
1445 {
1446 case MSR_K8_LSTAR:
1447 case MSR_K6_STAR:
1448 case MSR_K8_SF_MASK:
1449 case MSR_K8_KERNEL_GS_BASE:
1450 return true;
1451 }
1452 return false;
1453}
1454
1455
1456/**
1457 * Saves a set of guests MSRs back into the guest-CPU context.
1458 *
1459 * @param pVCpu Pointer to the VMCPU.
1460 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1461 * out-of-sync. Make sure to update the required fields
1462 * before using them.
1463 *
1464 * @remarks No-long-jump zone!!!
1465 */
1466static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1467{
1468 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1469 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1470
1471 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1472 {
1473 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1474 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1475 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1476 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1477 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1478 }
1479}
1480
1481
1482/**
1483 * Loads a set of guests MSRs to allow read/passthru to the guest.
1484 *
1485 * The name of this function is slightly confusing. This function does NOT
1486 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1487 * common prefix for functions dealing with "lazy restoration" of the shared
1488 * MSRs.
1489 *
1490 * @param pVCpu Pointer to the VMCPU.
1491 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1492 * out-of-sync. Make sure to update the required fields
1493 * before using them.
1494 *
1495 * @remarks No-long-jump zone!!!
1496 */
1497static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1498{
1499 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1500 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1501
1502 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1503 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1504 {
1505#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1506 do { \
1507 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1508 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1509 else \
1510 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1511 } while (0)
1512
1513 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1514 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1515 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1516 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1517#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1518 }
1519 else
1520 {
1521 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1522 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1523 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1524 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1525 }
1526 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1527}
1528
1529
1530/**
1531 * Performs lazy restoration of the set of host MSRs if they were previously
1532 * loaded with guest MSR values.
1533 *
1534 * @param pVCpu Pointer to the VMCPU.
1535 *
1536 * @remarks No-long-jump zone!!!
1537 * @remarks The guest MSRs should have been saved back into the guest-CPU
1538 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1539 */
1540static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1541{
1542 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1543 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1544
1545 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1546 {
1547 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1548 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1549 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1550 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1551 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1552 }
1553 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1554}
1555#endif /* HC_ARCH_BITS == 64 */
1556
1557
1558#ifdef VBOX_STRICT
1559/**
1560 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1561 * VMCS are correct.
1562 *
1563 * @param pVCpu Pointer to the VMCPU.
1564 */
1565static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1566{
1567 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1568
1569 /* Verify MSR counts in the VMCS are what we think it should be. */
1570 uint32_t cMsrs;
1571 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1572 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1573
1574 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1575 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1576
1577 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1578 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1579
1580 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1581 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1582 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1583 {
1584 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1585 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1586 pGuestMsr->u32Msr));
1587
1588 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1589 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1590 pHostMsr->u64Value, u64Msr));
1591
1592 /* Verify that the permissions are as expected in the MSR bitmap. */
1593 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1594 {
1595 VMXMSREXITREAD enmRead;
1596 VMXMSREXITWRITE enmWrite;
1597 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1598 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1599 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1600 pGuestMsr->u32Msr));
1601 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1602 pGuestMsr->u32Msr));
1603 }
1604 }
1605}
1606# endif /* VBOX_STRICT */
1607
1608
1609/**
1610 * Flushes the TLB using EPT.
1611 *
1612 * @returns VBox status code.
1613 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1614 * enmFlush).
1615 * @param enmFlush Type of flush.
1616 *
1617 * @remarks Caller is responsible for making sure this function is called only
1618 * when NestedPaging is supported and providing @a enmFlush that is
1619 * supported by the CPU.
1620 * @remarks Can be called with interrupts disabled.
1621 */
1622static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1623{
1624 uint64_t au64Descriptor[2];
1625 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1626 au64Descriptor[0] = 0;
1627 else
1628 {
1629 Assert(pVCpu);
1630 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1631 }
1632 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1633
1634 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1635 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1636 rc));
1637 if ( RT_SUCCESS(rc)
1638 && pVCpu)
1639 {
1640 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1641 }
1642}
1643
1644
1645/**
1646 * Flushes the TLB using VPID.
1647 *
1648 * @returns VBox status code.
1649 * @param pVM Pointer to the VM.
1650 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1651 * enmFlush).
1652 * @param enmFlush Type of flush.
1653 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1654 * on @a enmFlush).
1655 *
1656 * @remarks Can be called with interrupts disabled.
1657 */
1658static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1659{
1660 NOREF(pVM);
1661 AssertPtr(pVM);
1662 Assert(pVM->hm.s.vmx.fVpid);
1663
1664 uint64_t au64Descriptor[2];
1665 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1666 {
1667 au64Descriptor[0] = 0;
1668 au64Descriptor[1] = 0;
1669 }
1670 else
1671 {
1672 AssertPtr(pVCpu);
1673 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1674 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1675 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1676 au64Descriptor[1] = GCPtr;
1677 }
1678
1679 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1680 AssertMsg(rc == VINF_SUCCESS,
1681 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1682 if ( RT_SUCCESS(rc)
1683 && pVCpu)
1684 {
1685 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1686 }
1687}
1688
1689
1690/**
1691 * Invalidates a guest page by guest virtual address. Only relevant for
1692 * EPT/VPID, otherwise there is nothing really to invalidate.
1693 *
1694 * @returns VBox status code.
1695 * @param pVM Pointer to the VM.
1696 * @param pVCpu Pointer to the VMCPU.
1697 * @param GCVirt Guest virtual address of the page to invalidate.
1698 */
1699VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1700{
1701 AssertPtr(pVM);
1702 AssertPtr(pVCpu);
1703 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1704
1705 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1706 if (!fFlushPending)
1707 {
1708 /*
1709 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1710 * See @bugref{6043} and @bugref{6177}.
1711 *
1712 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1713 * function maybe called in a loop with individual addresses.
1714 */
1715 if (pVM->hm.s.vmx.fVpid)
1716 {
1717 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1718 {
1719 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1720 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1721 }
1722 else
1723 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1724 }
1725 else if (pVM->hm.s.fNestedPaging)
1726 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1727 }
1728
1729 return VINF_SUCCESS;
1730}
1731
1732
1733/**
1734 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1735 * otherwise there is nothing really to invalidate.
1736 *
1737 * @returns VBox status code.
1738 * @param pVM Pointer to the VM.
1739 * @param pVCpu Pointer to the VMCPU.
1740 * @param GCPhys Guest physical address of the page to invalidate.
1741 */
1742VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1743{
1744 NOREF(pVM); NOREF(GCPhys);
1745 LogFlowFunc(("%RGp\n", GCPhys));
1746
1747 /*
1748 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1749 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1750 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1751 */
1752 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1753 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1754 return VINF_SUCCESS;
1755}
1756
1757
1758/**
1759 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1760 * case where neither EPT nor VPID is supported by the CPU.
1761 *
1762 * @param pVM Pointer to the VM.
1763 * @param pVCpu Pointer to the VMCPU.
1764 * @param pCpu Pointer to the global HM struct.
1765 *
1766 * @remarks Called with interrupts disabled.
1767 */
1768static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1769{
1770 AssertPtr(pVCpu);
1771 AssertPtr(pCpu);
1772 NOREF(pVM);
1773
1774 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1775
1776 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1777#if 0
1778 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1779 pVCpu->hm.s.TlbShootdown.cPages = 0;
1780#endif
1781
1782 Assert(pCpu->idCpu != NIL_RTCPUID);
1783 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1784 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1785 pVCpu->hm.s.fForceTLBFlush = false;
1786 return;
1787}
1788
1789
1790/**
1791 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1792 *
1793 * @param pVM Pointer to the VM.
1794 * @param pVCpu Pointer to the VMCPU.
1795 * @param pCpu Pointer to the global HM CPU struct.
1796 * @remarks All references to "ASID" in this function pertains to "VPID" in
1797 * Intel's nomenclature. The reason is, to avoid confusion in compare
1798 * statements since the host-CPU copies are named "ASID".
1799 *
1800 * @remarks Called with interrupts disabled.
1801 */
1802static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1803{
1804#ifdef VBOX_WITH_STATISTICS
1805 bool fTlbFlushed = false;
1806# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1807# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1808 if (!fTlbFlushed) \
1809 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1810 } while (0)
1811#else
1812# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1813# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1814#endif
1815
1816 AssertPtr(pVM);
1817 AssertPtr(pCpu);
1818 AssertPtr(pVCpu);
1819 Assert(pCpu->idCpu != NIL_RTCPUID);
1820
1821 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1822 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1823 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1824
1825 /*
1826 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1827 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1828 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1829 */
1830 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1831 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1832 {
1833 ++pCpu->uCurrentAsid;
1834 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1835 {
1836 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1837 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1838 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1839 }
1840
1841 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1842 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1843 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1844
1845 /*
1846 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1847 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1848 */
1849 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1850 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1851 HMVMX_SET_TAGGED_TLB_FLUSHED();
1852 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1853 }
1854
1855 /* Check for explicit TLB shootdowns. */
1856 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1857 {
1858 /*
1859 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1860 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1861 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1862 * but not guest-physical mappings.
1863 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1864 */
1865 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1866 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1867 HMVMX_SET_TAGGED_TLB_FLUSHED();
1868 }
1869
1870 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1871 * where it is commented out. Support individual entry flushing
1872 * someday. */
1873#if 0
1874 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1875 {
1876 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1877
1878 /*
1879 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1880 * as supported by the CPU.
1881 */
1882 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1883 {
1884 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1885 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1886 }
1887 else
1888 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1889
1890 HMVMX_SET_TAGGED_TLB_FLUSHED();
1891 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1892 pVCpu->hm.s.TlbShootdown.cPages = 0;
1893 }
1894#endif
1895
1896 pVCpu->hm.s.fForceTLBFlush = false;
1897
1898 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1899
1900 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1901 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1902 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1903 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1904 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1905 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1906 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1907 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1908 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1909
1910 /* Update VMCS with the VPID. */
1911 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1912 AssertRC(rc);
1913
1914#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1915}
1916
1917
1918/**
1919 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1920 *
1921 * @returns VBox status code.
1922 * @param pVM Pointer to the VM.
1923 * @param pVCpu Pointer to the VMCPU.
1924 * @param pCpu Pointer to the global HM CPU struct.
1925 *
1926 * @remarks Called with interrupts disabled.
1927 */
1928static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1929{
1930 AssertPtr(pVM);
1931 AssertPtr(pVCpu);
1932 AssertPtr(pCpu);
1933 Assert(pCpu->idCpu != NIL_RTCPUID);
1934 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1935 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1936
1937 /*
1938 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1939 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1940 */
1941 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1942 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1943 {
1944 pVCpu->hm.s.fForceTLBFlush = true;
1945 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946 }
1947
1948 /* Check for explicit TLB shootdown flushes. */
1949 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1950 {
1951 pVCpu->hm.s.fForceTLBFlush = true;
1952 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1953 }
1954
1955 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958 if (pVCpu->hm.s.fForceTLBFlush)
1959 {
1960 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961 pVCpu->hm.s.fForceTLBFlush = false;
1962 }
1963 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1964 * where it is commented out. Support individual entry flushing
1965 * someday. */
1966#if 0
1967 else
1968 {
1969 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1970 {
1971 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1973 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1974 }
1975 else
1976 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1977
1978 pVCpu->hm.s.TlbShootdown.cPages = 0;
1979 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980 }
1981#endif
1982}
1983
1984
1985/**
1986 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1987 *
1988 * @returns VBox status code.
1989 * @param pVM Pointer to the VM.
1990 * @param pVCpu Pointer to the VMCPU.
1991 * @param pCpu Pointer to the global HM CPU struct.
1992 *
1993 * @remarks Called with interrupts disabled.
1994 */
1995static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1996{
1997 AssertPtr(pVM);
1998 AssertPtr(pVCpu);
1999 AssertPtr(pCpu);
2000 Assert(pCpu->idCpu != NIL_RTCPUID);
2001 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2002 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2003
2004 /*
2005 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2006 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2007 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2008 */
2009 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2010 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2011 {
2012 pVCpu->hm.s.fForceTLBFlush = true;
2013 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2014 }
2015
2016 /* Check for explicit TLB shootdown flushes. */
2017 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2018 {
2019 /*
2020 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2021 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2022 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2023 */
2024 pVCpu->hm.s.fForceTLBFlush = true;
2025 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2026 }
2027
2028 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2029 if (pVCpu->hm.s.fForceTLBFlush)
2030 {
2031 ++pCpu->uCurrentAsid;
2032 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2033 {
2034 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2035 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2036 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2037 }
2038
2039 pVCpu->hm.s.fForceTLBFlush = false;
2040 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2041 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2042 if (pCpu->fFlushAsidBeforeUse)
2043 {
2044 if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_SINGLE_CONTEXT)
2045 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2046 else if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_ALL_CONTEXTS)
2047 {
2048 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
2049 pCpu->fFlushAsidBeforeUse = false;
2050 }
2051 else
2052 {
2053 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2054 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2055 }
2056 }
2057 }
2058 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2059 * where it is commented out. Support individual entry flushing
2060 * someday. */
2061#if 0
2062 else
2063 {
2064 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2065 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2066 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2067 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2068
2069 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2070 {
2071 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2072 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2073 {
2074 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2075 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2076 }
2077 else
2078 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2079
2080 pVCpu->hm.s.TlbShootdown.cPages = 0;
2081 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2082 }
2083 else
2084 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2085 }
2086#endif
2087
2088 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2089 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2090 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2091 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2092 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2093 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2094 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2095
2096 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2097 AssertRC(rc);
2098}
2099
2100
2101/**
2102 * Flushes the guest TLB entry based on CPU capabilities.
2103 *
2104 * @param pVCpu Pointer to the VMCPU.
2105 * @param pCpu Pointer to the global HM CPU struct.
2106 */
2107DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2108{
2109#ifdef HMVMX_ALWAYS_FLUSH_TLB
2110 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2111#endif
2112 PVM pVM = pVCpu->CTX_SUFF(pVM);
2113 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2114 {
2115 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2116 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2117 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2118 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2119 default:
2120 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2121 break;
2122 }
2123
2124 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2125 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2126
2127 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2128}
2129
2130
2131/**
2132 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2133 * TLB entries from the host TLB before VM-entry.
2134 *
2135 * @returns VBox status code.
2136 * @param pVM Pointer to the VM.
2137 */
2138static int hmR0VmxSetupTaggedTlb(PVM pVM)
2139{
2140 /*
2141 * Determine optimal flush type for Nested Paging.
2142 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2143 * guest execution (see hmR3InitFinalizeR0()).
2144 */
2145 if (pVM->hm.s.fNestedPaging)
2146 {
2147 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2148 {
2149 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2150 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2151 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2152 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2153 else
2154 {
2155 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2156 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2157 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2158 }
2159
2160 /* Make sure the write-back cacheable memory type for EPT is supported. */
2161 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2162 {
2163 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2164 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2165 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2166 }
2167 }
2168 else
2169 {
2170 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2171 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2172 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2173 }
2174 }
2175
2176 /*
2177 * Determine optimal flush type for VPID.
2178 */
2179 if (pVM->hm.s.vmx.fVpid)
2180 {
2181 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2182 {
2183 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2184 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2185 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2186 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2187 else
2188 {
2189 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2190 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2191 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2192 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2193 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2194 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2195 pVM->hm.s.vmx.fVpid = false;
2196 }
2197 }
2198 else
2199 {
2200 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2201 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2202 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2203 pVM->hm.s.vmx.fVpid = false;
2204 }
2205 }
2206
2207 /*
2208 * Setup the handler for flushing tagged-TLBs.
2209 */
2210 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2211 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2212 else if (pVM->hm.s.fNestedPaging)
2213 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2214 else if (pVM->hm.s.vmx.fVpid)
2215 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2216 else
2217 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2218 return VINF_SUCCESS;
2219}
2220
2221
2222/**
2223 * Sets up pin-based VM-execution controls in the VMCS.
2224 *
2225 * @returns VBox status code.
2226 * @param pVM Pointer to the VM.
2227 * @param pVCpu Pointer to the VMCPU.
2228 */
2229static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2230{
2231 AssertPtr(pVM);
2232 AssertPtr(pVCpu);
2233
2234 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2235 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2236
2237 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2238 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2239 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2240
2241 /* Enable the VMX preemption timer. */
2242 if (pVM->hm.s.vmx.fUsePreemptTimer)
2243 {
2244 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2245 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2246 }
2247
2248 if ((val & zap) != val)
2249 {
2250 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2251 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2252 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255
2256 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2257 AssertRCReturn(rc, rc);
2258
2259 /* Update VCPU with the currently set pin-based VM-execution controls. */
2260 pVCpu->hm.s.vmx.u32PinCtls = val;
2261 return rc;
2262}
2263
2264
2265/**
2266 * Sets up processor-based VM-execution controls in the VMCS.
2267 *
2268 * @returns VBox status code.
2269 * @param pVM Pointer to the VM.
2270 * @param pVMCPU Pointer to the VMCPU.
2271 */
2272static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2273{
2274 AssertPtr(pVM);
2275 AssertPtr(pVCpu);
2276
2277 int rc = VERR_INTERNAL_ERROR_5;
2278 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2279 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2280
2281 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2282 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2283 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2284 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2285 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2286 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2287 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2288
2289 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2290 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2291 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2292 {
2293 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2294 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2295 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2296 }
2297
2298 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2299 if (!pVM->hm.s.fNestedPaging)
2300 {
2301 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2302 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2303 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2304 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2305 }
2306
2307 /* Use TPR shadowing if supported by the CPU. */
2308 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2309 {
2310 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2311 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2312 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2313 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2314 AssertRCReturn(rc, rc);
2315
2316 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2317 /* CR8 writes causes a VM-exit based on TPR threshold. */
2318 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2319 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2320 }
2321 else
2322 {
2323 /*
2324 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2325 * Set this control only for 64-bit guests.
2326 */
2327 if (pVM->hm.s.fAllow64BitGuests)
2328 {
2329 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2330 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2331 }
2332 }
2333
2334 /* Use MSR-bitmaps if supported by the CPU. */
2335 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2336 {
2337 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2338
2339 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2340 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2341 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2342 AssertRCReturn(rc, rc);
2343
2344 /*
2345 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2346 * automatically as dedicated fields in the VMCS.
2347 */
2348 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2349 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2350 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2351 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2352 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2353
2354#if HC_ARCH_BITS == 64
2355 /*
2356 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2357 */
2358 if (pVM->hm.s.fAllow64BitGuests)
2359 {
2360 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2361 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2362 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2363 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2364 }
2365#endif
2366 }
2367
2368 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2369 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2370 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2371
2372 if ((val & zap) != val)
2373 {
2374 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2375 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2376 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2377 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2378 }
2379
2380 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2381 AssertRCReturn(rc, rc);
2382
2383 /* Update VCPU with the currently set processor-based VM-execution controls. */
2384 pVCpu->hm.s.vmx.u32ProcCtls = val;
2385
2386 /*
2387 * Secondary processor-based VM-execution controls.
2388 */
2389 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2390 {
2391 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2392 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2393
2394 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2395 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2396
2397 if (pVM->hm.s.fNestedPaging)
2398 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2399 else
2400 {
2401 /*
2402 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2403 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2404 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2405 */
2406 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2407 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2408 }
2409
2410 if (pVM->hm.s.vmx.fVpid)
2411 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2412
2413 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2414 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2415
2416 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2417 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2418 * done dynamically. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2420 {
2421 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2422 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2423 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2424 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2425 AssertRCReturn(rc, rc);
2426 }
2427
2428 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2429 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2430
2431 if ((val & zap) != val)
2432 {
2433 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2434 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2435 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2436 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2437 }
2438
2439 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2440 AssertRCReturn(rc, rc);
2441
2442 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2443 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2444 }
2445 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2446 {
2447 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2448 "available\n"));
2449 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2450 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2451 }
2452
2453 return VINF_SUCCESS;
2454}
2455
2456
2457/**
2458 * Sets up miscellaneous (everything other than Pin & Processor-based
2459 * VM-execution) control fields in the VMCS.
2460 *
2461 * @returns VBox status code.
2462 * @param pVM Pointer to the VM.
2463 * @param pVCpu Pointer to the VMCPU.
2464 */
2465static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2466{
2467 NOREF(pVM);
2468 AssertPtr(pVM);
2469 AssertPtr(pVCpu);
2470
2471 int rc = VERR_GENERAL_FAILURE;
2472
2473 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2474#if 0
2475 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2477 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2478
2479 /*
2480 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2481 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2482 * We thus use the exception bitmap to control it rather than use both.
2483 */
2484 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2485 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2486
2487 /** @todo Explore possibility of using IO-bitmaps. */
2488 /* All IO & IOIO instructions cause VM-exits. */
2489 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2490 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2491
2492 /* Initialize the MSR-bitmap area. */
2493 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2494 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2495 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2496#endif
2497
2498 /* Setup MSR auto-load/store area. */
2499 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2500 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2501 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2502 AssertRCReturn(rc, rc);
2503 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2504 AssertRCReturn(rc, rc);
2505
2506 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2507 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2508 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2509 AssertRCReturn(rc, rc);
2510
2511 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2512 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2513 AssertRCReturn(rc, rc);
2514
2515 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2516#if 0
2517 /* Setup debug controls */
2518 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2519 AssertRCReturn(rc, rc);
2520 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2521 AssertRCReturn(rc, rc);
2522#endif
2523
2524 return rc;
2525}
2526
2527
2528/**
2529 * Sets up the initial exception bitmap in the VMCS based on static conditions
2530 * (i.e. conditions that cannot ever change after starting the VM).
2531 *
2532 * @returns VBox status code.
2533 * @param pVM Pointer to the VM.
2534 * @param pVCpu Pointer to the VMCPU.
2535 */
2536static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2537{
2538 AssertPtr(pVM);
2539 AssertPtr(pVCpu);
2540
2541 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2542
2543 uint32_t u32XcptBitmap = 0;
2544
2545 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2546 if (!pVM->hm.s.fNestedPaging)
2547 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2548
2549 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2550 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2551 AssertRCReturn(rc, rc);
2552 return rc;
2553}
2554
2555
2556/**
2557 * Sets up the initial guest-state mask. The guest-state mask is consulted
2558 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2559 * for the nested virtualization case (as it would cause a VM-exit).
2560 *
2561 * @param pVCpu Pointer to the VMCPU.
2562 */
2563static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2564{
2565 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2566 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2567 return VINF_SUCCESS;
2568}
2569
2570
2571/**
2572 * Does per-VM VT-x initialization.
2573 *
2574 * @returns VBox status code.
2575 * @param pVM Pointer to the VM.
2576 */
2577VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2578{
2579 LogFlowFunc(("pVM=%p\n", pVM));
2580
2581 int rc = hmR0VmxStructsAlloc(pVM);
2582 if (RT_FAILURE(rc))
2583 {
2584 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2585 return rc;
2586 }
2587
2588 return VINF_SUCCESS;
2589}
2590
2591
2592/**
2593 * Does per-VM VT-x termination.
2594 *
2595 * @returns VBox status code.
2596 * @param pVM Pointer to the VM.
2597 */
2598VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2599{
2600 LogFlowFunc(("pVM=%p\n", pVM));
2601
2602#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2603 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2604 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2605#endif
2606 hmR0VmxStructsFree(pVM);
2607 return VINF_SUCCESS;
2608}
2609
2610
2611/**
2612 * Sets up the VM for execution under VT-x.
2613 * This function is only called once per-VM during initialization.
2614 *
2615 * @returns VBox status code.
2616 * @param pVM Pointer to the VM.
2617 */
2618VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2619{
2620 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2621 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2622
2623 LogFlowFunc(("pVM=%p\n", pVM));
2624
2625 /*
2626 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2627 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2628 */
2629 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2630 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2631 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2632 || !pVM->hm.s.vmx.pRealModeTSS))
2633 {
2634 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2635 return VERR_INTERNAL_ERROR;
2636 }
2637
2638#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2639 /*
2640 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2641 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2642 */
2643 if ( pVM->hm.s.fAllow64BitGuests
2644 && !HMVMX_IS_64BIT_HOST_MODE())
2645 {
2646 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2647 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2648 }
2649#endif
2650
2651 /* Initialize these always, see hmR3InitFinalizeR0().*/
2652 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2653 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2654
2655 /* Setup the tagged-TLB flush handlers. */
2656 int rc = hmR0VmxSetupTaggedTlb(pVM);
2657 if (RT_FAILURE(rc))
2658 {
2659 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2660 return rc;
2661 }
2662
2663 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2664 {
2665 PVMCPU pVCpu = &pVM->aCpus[i];
2666 AssertPtr(pVCpu);
2667 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2668
2669 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2670 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2671
2672 /* Set revision dword at the beginning of the VMCS structure. */
2673 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2674
2675 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2676 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2677 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2678 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2679
2680 /* Load this VMCS as the current VMCS. */
2681 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2682 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2683 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2684
2685 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2686 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2687 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2688
2689 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2690 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2694 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2695 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2696
2697 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2698 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2699 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2700
2701 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2702 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2703 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2704
2705#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2706 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2707 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2708 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2709#endif
2710
2711 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2712 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2713 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2714 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2715
2716 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2717
2718 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2719 }
2720
2721 return VINF_SUCCESS;
2722}
2723
2724
2725/**
2726 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2727 * the VMCS.
2728 *
2729 * @returns VBox status code.
2730 * @param pVM Pointer to the VM.
2731 * @param pVCpu Pointer to the VMCPU.
2732 */
2733DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2734{
2735 NOREF(pVM); NOREF(pVCpu);
2736
2737 RTCCUINTREG uReg = ASMGetCR0();
2738 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2739 AssertRCReturn(rc, rc);
2740
2741#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2742 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2743 if (HMVMX_IS_64BIT_HOST_MODE())
2744 {
2745 uint64_t uRegCR3 = HMR0Get64bitCR3();
2746 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2747 }
2748 else
2749#endif
2750 {
2751 uReg = ASMGetCR3();
2752 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2753 }
2754 AssertRCReturn(rc, rc);
2755
2756 uReg = ASMGetCR4();
2757 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2758 AssertRCReturn(rc, rc);
2759 return rc;
2760}
2761
2762
2763#if HC_ARCH_BITS == 64
2764/**
2765 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2766 * requirements. See hmR0VmxSaveHostSegmentRegs().
2767 */
2768# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2769 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2770 { \
2771 bool fValidSelector = true; \
2772 if ((selValue) & X86_SEL_LDT) \
2773 { \
2774 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2775 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2776 } \
2777 if (fValidSelector) \
2778 { \
2779 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2780 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2781 } \
2782 (selValue) = 0; \
2783 }
2784#endif
2785
2786
2787/**
2788 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2789 * the host-state area in the VMCS.
2790 *
2791 * @returns VBox status code.
2792 * @param pVM Pointer to the VM.
2793 * @param pVCpu Pointer to the VMCPU.
2794 */
2795DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2796{
2797 NOREF(pVM);
2798 int rc = VERR_INTERNAL_ERROR_5;
2799
2800#if HC_ARCH_BITS == 64
2801 /*
2802 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2803 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2804 */
2805 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2806 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2807#endif
2808
2809 /*
2810 * Host DS, ES, FS and GS segment registers.
2811 */
2812#if HC_ARCH_BITS == 64
2813 RTSEL uSelDS = ASMGetDS();
2814 RTSEL uSelES = ASMGetES();
2815 RTSEL uSelFS = ASMGetFS();
2816 RTSEL uSelGS = ASMGetGS();
2817#else
2818 RTSEL uSelDS = 0;
2819 RTSEL uSelES = 0;
2820 RTSEL uSelFS = 0;
2821 RTSEL uSelGS = 0;
2822#endif
2823
2824 /* Recalculate which host-state bits need to be manually restored. */
2825 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2826
2827 /*
2828 * Host CS and SS segment registers.
2829 */
2830#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2831 RTSEL uSelCS;
2832 RTSEL uSelSS;
2833 if (HMVMX_IS_64BIT_HOST_MODE())
2834 {
2835 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2836 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2837 }
2838 else
2839 {
2840 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2841 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2842 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2843 }
2844#else
2845 RTSEL uSelCS = ASMGetCS();
2846 RTSEL uSelSS = ASMGetSS();
2847#endif
2848
2849 /*
2850 * Host TR segment register.
2851 */
2852 RTSEL uSelTR = ASMGetTR();
2853
2854#if HC_ARCH_BITS == 64
2855 /*
2856 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2857 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2858 */
2859 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2860 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2861 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2862 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2863# undef VMXLOCAL_ADJUST_HOST_SEG
2864#endif
2865
2866 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2867 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2868 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2869 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2870 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2871 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2872 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2873 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2874 Assert(uSelCS);
2875 Assert(uSelTR);
2876
2877 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2878#if 0
2879 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2880 Assert(uSelSS != 0);
2881#endif
2882
2883 /* Write these host selector fields into the host-state area in the VMCS. */
2884 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2885 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2886#if HC_ARCH_BITS == 64
2887 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2888 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2889 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2890 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2891#endif
2892 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2893
2894 /*
2895 * Host GDTR and IDTR.
2896 */
2897 RTGDTR Gdtr;
2898 RT_ZERO(Gdtr);
2899#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2900 if (HMVMX_IS_64BIT_HOST_MODE())
2901 {
2902 X86XDTR64 Gdtr64;
2903 X86XDTR64 Idtr64;
2904 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2905 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2906 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2907
2908 Gdtr.cbGdt = Gdtr64.cb;
2909 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2910 }
2911 else
2912#endif
2913 {
2914 RTIDTR Idtr;
2915 ASMGetGDTR(&Gdtr);
2916 ASMGetIDTR(&Idtr);
2917 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2918 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2919
2920#if HC_ARCH_BITS == 64
2921 /*
2922 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2923 * maximum limit (0xffff) on every VM-exit.
2924 */
2925 if (Gdtr.cbGdt != 0xffff)
2926 {
2927 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2928 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2929 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2930 }
2931
2932 /*
2933 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2934 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2935 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2936 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2937 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2938 * hosts where we are pretty sure it won't cause trouble.
2939 */
2940# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2941 if (Idtr.cbIdt < 0x0fff)
2942# else
2943 if (Idtr.cbIdt != 0xffff)
2944# endif
2945 {
2946 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2947 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2948 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2949 }
2950#endif
2951 }
2952
2953 /*
2954 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2955 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2956 */
2957 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
2958 {
2959 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2960 return VERR_VMX_INVALID_HOST_STATE;
2961 }
2962
2963 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2964#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2965 if (HMVMX_IS_64BIT_HOST_MODE())
2966 {
2967 /* We need the 64-bit TR base for hybrid darwin. */
2968 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2969 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2970 }
2971 else
2972#endif
2973 {
2974 uintptr_t uTRBase;
2975#if HC_ARCH_BITS == 64
2976 uTRBase = X86DESC64_BASE(pDesc);
2977
2978 /*
2979 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2980 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2981 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2982 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2983 *
2984 * [1] See Intel spec. 3.5 "System Descriptor Types".
2985 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2986 */
2987 Assert(pDesc->System.u4Type == 11);
2988 if ( pDesc->System.u16LimitLow != 0x67
2989 || pDesc->System.u4LimitHigh)
2990 {
2991 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2992 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2993
2994 /* Store the GDTR here as we need it while restoring TR. */
2995 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2996 }
2997#else
2998 uTRBase = X86DESC_BASE(pDesc);
2999#endif
3000 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3001 }
3002 AssertRCReturn(rc, rc);
3003
3004 /*
3005 * Host FS base and GS base.
3006 */
3007#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3011 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3012 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3014
3015# if HC_ARCH_BITS == 64
3016 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3017 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3018 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3019 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3020 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3021# endif
3022 }
3023#endif
3024 return rc;
3025}
3026
3027
3028/**
3029 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3030 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3031 * the host after every successful VM-exit.
3032 *
3033 * @returns VBox status code.
3034 * @param pVM Pointer to the VM.
3035 * @param pVCpu Pointer to the VMCPU.
3036 *
3037 * @remarks No-long-jump zone!!!
3038 */
3039DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3040{
3041 NOREF(pVM);
3042
3043 AssertPtr(pVCpu);
3044 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3045
3046 int rc = VINF_SUCCESS;
3047#if HC_ARCH_BITS == 64
3048 if (pVM->hm.s.fAllow64BitGuests)
3049 hmR0VmxLazySaveHostMsrs(pVCpu);
3050#endif
3051
3052 /*
3053 * Host Sysenter MSRs.
3054 */
3055 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3056 AssertRCReturn(rc, rc);
3057#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3058 if (HMVMX_IS_64BIT_HOST_MODE())
3059 {
3060 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3061 AssertRCReturn(rc, rc);
3062 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3063 }
3064 else
3065 {
3066 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3067 AssertRCReturn(rc, rc);
3068 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3069 }
3070#elif HC_ARCH_BITS == 32
3071 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3072 AssertRCReturn(rc, rc);
3073 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3074#else
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3076 AssertRCReturn(rc, rc);
3077 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3078#endif
3079 AssertRCReturn(rc, rc);
3080
3081 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3082 * hmR0VmxSetupExitCtls() !! */
3083 return rc;
3084}
3085
3086
3087/**
3088 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3089 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3090 * controls".
3091 *
3092 * @returns VBox status code.
3093 * @param pVCpu Pointer to the VMCPU.
3094 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3095 * out-of-sync. Make sure to update the required fields
3096 * before using them.
3097 *
3098 * @remarks No-long-jump zone!!!
3099 */
3100DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3101{
3102 int rc = VINF_SUCCESS;
3103 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3104 {
3105 PVM pVM = pVCpu->CTX_SUFF(pVM);
3106 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3107 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3108
3109 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3110 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3111
3112 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3113 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3114 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3115 else
3116 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3117
3118 /*
3119 * The following should -not- be set (since we're not in SMM mode):
3120 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3121 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3122 */
3123
3124 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3125 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3126 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3127
3128 if ((val & zap) != val)
3129 {
3130 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3131 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3132 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3133 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3134 }
3135
3136 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3137 AssertRCReturn(rc, rc);
3138
3139 /* Update VCPU with the currently set VM-exit controls. */
3140 pVCpu->hm.s.vmx.u32EntryCtls = val;
3141 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3142 }
3143 return rc;
3144}
3145
3146
3147/**
3148 * Sets up the VM-exit controls in the VMCS.
3149 *
3150 * @returns VBox status code.
3151 * @param pVM Pointer to the VM.
3152 * @param pVCpu Pointer to the VMCPU.
3153 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3154 * out-of-sync. Make sure to update the required fields
3155 * before using them.
3156 *
3157 * @remarks requires EFER.
3158 */
3159DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3160{
3161 NOREF(pMixedCtx);
3162
3163 int rc = VINF_SUCCESS;
3164 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3165 {
3166 PVM pVM = pVCpu->CTX_SUFF(pVM);
3167 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3168 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3169
3170 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3171 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3172
3173 /*
3174 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3175 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3176 */
3177#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3178 if (HMVMX_IS_64BIT_HOST_MODE())
3179 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3180 else
3181 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3182#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3183 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3184 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3185 else
3186 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3187#endif
3188
3189 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3190 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3191
3192 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3193 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3194 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3195 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3196 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3197
3198 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3199 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3200
3201 if ((val & zap) != val)
3202 {
3203 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3204 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3205 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3206 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3207 }
3208
3209 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3210 AssertRCReturn(rc, rc);
3211
3212 /* Update VCPU with the currently set VM-exit controls. */
3213 pVCpu->hm.s.vmx.u32ExitCtls = val;
3214 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3215 }
3216 return rc;
3217}
3218
3219
3220/**
3221 * Loads the guest APIC and related state.
3222 *
3223 * @returns VBox status code.
3224 * @param pVM Pointer to the VM.
3225 * @param pVCpu Pointer to the VMCPU.
3226 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3227 * out-of-sync. Make sure to update the required fields
3228 * before using them.
3229 */
3230DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3231{
3232 NOREF(pMixedCtx);
3233
3234 int rc = VINF_SUCCESS;
3235 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3236 {
3237 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3238 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3239 {
3240 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3241
3242 bool fPendingIntr = false;
3243 uint8_t u8Tpr = 0;
3244 uint8_t u8PendingIntr = 0;
3245 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3246 AssertRCReturn(rc, rc);
3247
3248 /*
3249 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3250 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3251 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3252 * the interrupt when we VM-exit for other reasons.
3253 */
3254 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3255 uint32_t u32TprThreshold = 0;
3256 if (fPendingIntr)
3257 {
3258 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3259 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3260 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3261 if (u8PendingPriority <= u8TprPriority)
3262 u32TprThreshold = u8PendingPriority;
3263 else
3264 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3265 }
3266 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3267
3268 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3269 AssertRCReturn(rc, rc);
3270 }
3271
3272 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3273 }
3274 return rc;
3275}
3276
3277
3278/**
3279 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3280 *
3281 * @returns Guest's interruptibility-state.
3282 * @param pVCpu Pointer to the VMCPU.
3283 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3284 * out-of-sync. Make sure to update the required fields
3285 * before using them.
3286 *
3287 * @remarks No-long-jump zone!!!
3288 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3289 */
3290DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3291{
3292 /*
3293 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3294 * inhibit interrupts or clear any existing interrupt-inhibition.
3295 */
3296 uint32_t uIntrState = 0;
3297 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3298 {
3299 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3300 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3301 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3302 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3303 {
3304 /*
3305 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3306 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3307 */
3308 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3309 }
3310 else if (pMixedCtx->eflags.Bits.u1IF)
3311 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3312 else
3313 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3314 }
3315 return uIntrState;
3316}
3317
3318
3319/**
3320 * Loads the guest's interruptibility-state into the guest-state area in the
3321 * VMCS.
3322 *
3323 * @returns VBox status code.
3324 * @param pVCpu Pointer to the VMCPU.
3325 * @param uIntrState The interruptibility-state to set.
3326 */
3327static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3328{
3329 NOREF(pVCpu);
3330 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3331 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3332 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3333 AssertRCReturn(rc, rc);
3334 return rc;
3335}
3336
3337
3338/**
3339 * Loads the guest's RIP into the guest-state area in the VMCS.
3340 *
3341 * @returns VBox status code.
3342 * @param pVCpu Pointer to the VMCPU.
3343 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3344 * out-of-sync. Make sure to update the required fields
3345 * before using them.
3346 *
3347 * @remarks No-long-jump zone!!!
3348 */
3349static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3350{
3351 int rc = VINF_SUCCESS;
3352 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3353 {
3354 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3355 AssertRCReturn(rc, rc);
3356
3357 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3358 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3359 }
3360 return rc;
3361}
3362
3363
3364/**
3365 * Loads the guest's RSP into the guest-state area in the VMCS.
3366 *
3367 * @returns VBox status code.
3368 * @param pVCpu Pointer to the VMCPU.
3369 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3370 * out-of-sync. Make sure to update the required fields
3371 * before using them.
3372 *
3373 * @remarks No-long-jump zone!!!
3374 */
3375static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3376{
3377 int rc = VINF_SUCCESS;
3378 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3379 {
3380 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3381 AssertRCReturn(rc, rc);
3382
3383 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3384 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3385 }
3386 return rc;
3387}
3388
3389
3390/**
3391 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3392 *
3393 * @returns VBox status code.
3394 * @param pVCpu Pointer to the VMCPU.
3395 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3396 * out-of-sync. Make sure to update the required fields
3397 * before using them.
3398 *
3399 * @remarks No-long-jump zone!!!
3400 */
3401static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3402{
3403 int rc = VINF_SUCCESS;
3404 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3405 {
3406 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3407 Let us assert it as such and use 32-bit VMWRITE. */
3408 Assert(!(pMixedCtx->rflags.u64 >> 32));
3409 X86EFLAGS Eflags = pMixedCtx->eflags;
3410 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3411 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3412
3413 /*
3414 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3415 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3416 */
3417 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3418 {
3419 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3420 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3421 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3422 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3423 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3424 }
3425
3426 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3427 AssertRCReturn(rc, rc);
3428
3429 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3430 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3431 }
3432 return rc;
3433}
3434
3435
3436/**
3437 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3438 *
3439 * @returns VBox status code.
3440 * @param pVCpu Pointer to the VMCPU.
3441 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3442 * out-of-sync. Make sure to update the required fields
3443 * before using them.
3444 *
3445 * @remarks No-long-jump zone!!!
3446 */
3447DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3448{
3449 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3450 AssertRCReturn(rc, rc);
3451 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3452 AssertRCReturn(rc, rc);
3453 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3454 AssertRCReturn(rc, rc);
3455 return rc;
3456}
3457
3458
3459/**
3460 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3461 * CR0 is partially shared with the host and we have to consider the FPU bits.
3462 *
3463 * @returns VBox status code.
3464 * @param pVM Pointer to the VM.
3465 * @param pVCpu Pointer to the VMCPU.
3466 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3467 * out-of-sync. Make sure to update the required fields
3468 * before using them.
3469 *
3470 * @remarks No-long-jump zone!!!
3471 */
3472static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3473{
3474 /*
3475 * Guest CR0.
3476 * Guest FPU.
3477 */
3478 int rc = VINF_SUCCESS;
3479 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3480 {
3481 Assert(!(pMixedCtx->cr0 >> 32));
3482 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3483 PVM pVM = pVCpu->CTX_SUFF(pVM);
3484
3485 /* The guest's view (read access) of its CR0 is unblemished. */
3486 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3487 AssertRCReturn(rc, rc);
3488 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3489
3490 /* Setup VT-x's view of the guest CR0. */
3491 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3492 if (pVM->hm.s.fNestedPaging)
3493 {
3494 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3495 {
3496 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3497 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3498 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3499 }
3500 else
3501 {
3502 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3503 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3504 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3505 }
3506
3507 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3508 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3509 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3510
3511 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3512 AssertRCReturn(rc, rc);
3513 }
3514 else
3515 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3516
3517 /*
3518 * Guest FPU bits.
3519 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3520 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3521 */
3522 u32GuestCR0 |= X86_CR0_NE;
3523 bool fInterceptNM = false;
3524 if (CPUMIsGuestFPUStateActive(pVCpu))
3525 {
3526 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3527 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3528 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3529 }
3530 else
3531 {
3532 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3533 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3534 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3535 }
3536
3537 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3538 bool fInterceptMF = false;
3539 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3540 fInterceptMF = true;
3541
3542 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3543 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3544 {
3545 Assert(PDMVmmDevHeapIsEnabled(pVM));
3546 Assert(pVM->hm.s.vmx.pRealModeTSS);
3547 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3548 fInterceptNM = true;
3549 fInterceptMF = true;
3550 }
3551 else
3552 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3553
3554 if (fInterceptNM)
3555 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3556 else
3557 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3558
3559 if (fInterceptMF)
3560 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3561 else
3562 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3563
3564 /* Additional intercepts for debugging, define these yourself explicitly. */
3565#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3566 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3567 | RT_BIT(X86_XCPT_BP)
3568 | RT_BIT(X86_XCPT_DB)
3569 | RT_BIT(X86_XCPT_DE)
3570 | RT_BIT(X86_XCPT_NM)
3571 | RT_BIT(X86_XCPT_TS)
3572 | RT_BIT(X86_XCPT_UD)
3573 | RT_BIT(X86_XCPT_NP)
3574 | RT_BIT(X86_XCPT_SS)
3575 | RT_BIT(X86_XCPT_GP)
3576 | RT_BIT(X86_XCPT_PF)
3577 | RT_BIT(X86_XCPT_MF)
3578 ;
3579#elif defined(HMVMX_ALWAYS_TRAP_PF)
3580 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3581#endif
3582
3583 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3584
3585 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3586 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3587 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3588 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3589 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3590 else
3591 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3592
3593 u32GuestCR0 |= uSetCR0;
3594 u32GuestCR0 &= uZapCR0;
3595 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3596
3597 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3598 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3599 AssertRCReturn(rc, rc);
3600 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3601 AssertRCReturn(rc, rc);
3602 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3603
3604 /*
3605 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3606 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3607 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3608 */
3609 uint32_t u32CR0Mask = 0;
3610 u32CR0Mask = X86_CR0_PE
3611 | X86_CR0_NE
3612 | X86_CR0_WP
3613 | X86_CR0_PG
3614 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3615 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3616 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3617
3618 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3619 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3620 * and @bugref{6944}. */
3621#if 0
3622 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3623 u32CR0Mask &= ~X86_CR0_PE;
3624#endif
3625 if (pVM->hm.s.fNestedPaging)
3626 u32CR0Mask &= ~X86_CR0_WP;
3627
3628 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3629 if (fInterceptNM)
3630 {
3631 u32CR0Mask |= X86_CR0_TS
3632 | X86_CR0_MP;
3633 }
3634
3635 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3636 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3637 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3638 AssertRCReturn(rc, rc);
3639 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3640
3641 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3642 }
3643 return rc;
3644}
3645
3646
3647/**
3648 * Loads the guest control registers (CR3, CR4) into the guest-state area
3649 * in the VMCS.
3650 *
3651 * @returns VBox status code.
3652 * @param pVM Pointer to the VM.
3653 * @param pVCpu Pointer to the VMCPU.
3654 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3655 * out-of-sync. Make sure to update the required fields
3656 * before using them.
3657 *
3658 * @remarks No-long-jump zone!!!
3659 */
3660static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3661{
3662 int rc = VINF_SUCCESS;
3663 PVM pVM = pVCpu->CTX_SUFF(pVM);
3664
3665 /*
3666 * Guest CR2.
3667 * It's always loaded in the assembler code. Nothing to do here.
3668 */
3669
3670 /*
3671 * Guest CR3.
3672 */
3673 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3674 {
3675 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3676 if (pVM->hm.s.fNestedPaging)
3677 {
3678 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3679
3680 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3681 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3682 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3683 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3684
3685 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3686 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3687 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3688
3689 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3690 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3691 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3692 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3693
3694 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3695 AssertRCReturn(rc, rc);
3696 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3697
3698 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3699 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3700 {
3701 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3702 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3703 {
3704 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3705 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3706 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3707 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3708 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3709 }
3710
3711 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3712 have Unrestricted Execution to handle the guest when it's not using paging. */
3713 GCPhysGuestCR3 = pMixedCtx->cr3;
3714 }
3715 else
3716 {
3717 /*
3718 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3719 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3720 * EPT takes care of translating it to host-physical addresses.
3721 */
3722 RTGCPHYS GCPhys;
3723 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3724 Assert(PDMVmmDevHeapIsEnabled(pVM));
3725
3726 /* We obtain it here every time as the guest could have relocated this PCI region. */
3727 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3728 AssertRCReturn(rc, rc);
3729
3730 GCPhysGuestCR3 = GCPhys;
3731 }
3732
3733 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3734 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3735 }
3736 else
3737 {
3738 /* Non-nested paging case, just use the hypervisor's CR3. */
3739 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3740
3741 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3742 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3743 }
3744 AssertRCReturn(rc, rc);
3745
3746 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3747 }
3748
3749 /*
3750 * Guest CR4.
3751 */
3752 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3753 {
3754 Assert(!(pMixedCtx->cr4 >> 32));
3755 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3756
3757 /* The guest's view of its CR4 is unblemished. */
3758 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3759 AssertRCReturn(rc, rc);
3760 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3761
3762 /* Setup VT-x's view of the guest CR4. */
3763 /*
3764 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3765 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3766 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3767 */
3768 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3769 {
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 Assert(PDMVmmDevHeapIsEnabled(pVM));
3772 u32GuestCR4 &= ~X86_CR4_VME;
3773 }
3774
3775 if (pVM->hm.s.fNestedPaging)
3776 {
3777 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3778 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3779 {
3780 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3781 u32GuestCR4 |= X86_CR4_PSE;
3782 /* Our identity mapping is a 32-bit page directory. */
3783 u32GuestCR4 &= ~X86_CR4_PAE;
3784 }
3785 /* else use guest CR4.*/
3786 }
3787 else
3788 {
3789 /*
3790 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3791 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3792 */
3793 switch (pVCpu->hm.s.enmShadowMode)
3794 {
3795 case PGMMODE_REAL: /* Real-mode. */
3796 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3797 case PGMMODE_32_BIT: /* 32-bit paging. */
3798 {
3799 u32GuestCR4 &= ~X86_CR4_PAE;
3800 break;
3801 }
3802
3803 case PGMMODE_PAE: /* PAE paging. */
3804 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3805 {
3806 u32GuestCR4 |= X86_CR4_PAE;
3807 break;
3808 }
3809
3810 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3811 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3812#ifdef VBOX_ENABLE_64_BITS_GUESTS
3813 break;
3814#endif
3815 default:
3816 AssertFailed();
3817 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3818 }
3819 }
3820
3821 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3822 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3823 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3824 u32GuestCR4 |= uSetCR4;
3825 u32GuestCR4 &= uZapCR4;
3826
3827 /* Write VT-x's view of the guest CR4 into the VMCS. */
3828 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3829 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3830 AssertRCReturn(rc, rc);
3831
3832 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3833 uint32_t u32CR4Mask = 0;
3834 u32CR4Mask = X86_CR4_VME
3835 | X86_CR4_PAE
3836 | X86_CR4_PGE
3837 | X86_CR4_PSE
3838 | X86_CR4_VMXE;
3839 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3840 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3841 AssertRCReturn(rc, rc);
3842
3843 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3844 }
3845 return rc;
3846}
3847
3848
3849/**
3850 * Loads the guest debug registers into the guest-state area in the VMCS.
3851 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3852 *
3853 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3854 *
3855 * @returns VBox status code.
3856 * @param pVCpu Pointer to the VMCPU.
3857 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3858 * out-of-sync. Make sure to update the required fields
3859 * before using them.
3860 *
3861 * @remarks No-long-jump zone!!!
3862 */
3863static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3864{
3865 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3866 return VINF_SUCCESS;
3867
3868#ifdef VBOX_STRICT
3869 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3870 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3871 {
3872 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3873 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3874 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3875 }
3876#endif
3877
3878 int rc;
3879 PVM pVM = pVCpu->CTX_SUFF(pVM);
3880 bool fInterceptDB = false;
3881 bool fInterceptMovDRx = false;
3882 if ( pVCpu->hm.s.fSingleInstruction
3883 || DBGFIsStepping(pVCpu))
3884 {
3885 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3886 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3887 {
3888 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3889 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3890 AssertRCReturn(rc, rc);
3891 Assert(fInterceptDB == false);
3892 }
3893 else
3894 {
3895 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3896 pVCpu->hm.s.fClearTrapFlag = true;
3897 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3898 fInterceptDB = true;
3899 }
3900 }
3901
3902 if ( fInterceptDB
3903 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3904 {
3905 /*
3906 * Use the combined guest and host DRx values found in the hypervisor
3907 * register set because the debugger has breakpoints active or someone
3908 * is single stepping on the host side without a monitor trap flag.
3909 *
3910 * Note! DBGF expects a clean DR6 state before executing guest code.
3911 */
3912#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3913 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3914 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3915 {
3916 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3917 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3918 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3919 }
3920 else
3921#endif
3922 if (!CPUMIsHyperDebugStateActive(pVCpu))
3923 {
3924 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3925 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3926 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3927 }
3928
3929 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3930 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3931 AssertRCReturn(rc, rc);
3932
3933 pVCpu->hm.s.fUsingHyperDR7 = true;
3934 fInterceptDB = true;
3935 fInterceptMovDRx = true;
3936 }
3937 else
3938 {
3939 /*
3940 * If the guest has enabled debug registers, we need to load them prior to
3941 * executing guest code so they'll trigger at the right time.
3942 */
3943 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3944 {
3945#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3946 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3947 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3948 {
3949 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3950 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3951 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3952 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3953 }
3954 else
3955#endif
3956 if (!CPUMIsGuestDebugStateActive(pVCpu))
3957 {
3958 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3959 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3960 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3961 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3962 }
3963 Assert(!fInterceptDB);
3964 Assert(!fInterceptMovDRx);
3965 }
3966 /*
3967 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3968 * must intercept #DB in order to maintain a correct DR6 guest value.
3969 */
3970#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3971 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3972 && !CPUMIsGuestDebugStateActive(pVCpu))
3973#else
3974 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3975#endif
3976 {
3977 fInterceptMovDRx = true;
3978 fInterceptDB = true;
3979 }
3980
3981 /* Update guest DR7. */
3982 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3983 AssertRCReturn(rc, rc);
3984
3985 pVCpu->hm.s.fUsingHyperDR7 = false;
3986 }
3987
3988 /*
3989 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3990 */
3991 if (fInterceptDB)
3992 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3993 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3994 {
3995#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3996 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3997#endif
3998 }
3999 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4000 AssertRCReturn(rc, rc);
4001
4002 /*
4003 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4004 */
4005 if (fInterceptMovDRx)
4006 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4007 else
4008 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4009 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4010 AssertRCReturn(rc, rc);
4011
4012 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4013 return VINF_SUCCESS;
4014}
4015
4016
4017#ifdef VBOX_STRICT
4018/**
4019 * Strict function to validate segment registers.
4020 *
4021 * @remarks ASSUMES CR0 is up to date.
4022 */
4023static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4024{
4025 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4026 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4027 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4028 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4029 && ( !CPUMIsGuestInRealModeEx(pCtx)
4030 && !CPUMIsGuestInV86ModeEx(pCtx)))
4031 {
4032 /* Protected mode checks */
4033 /* CS */
4034 Assert(pCtx->cs.Attr.n.u1Present);
4035 Assert(!(pCtx->cs.Attr.u & 0xf00));
4036 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4037 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4038 || !(pCtx->cs.Attr.n.u1Granularity));
4039 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4040 || (pCtx->cs.Attr.n.u1Granularity));
4041 /* CS cannot be loaded with NULL in protected mode. */
4042 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4043 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4044 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4045 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4046 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4047 else
4048 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4049 /* SS */
4050 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4051 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4052 if ( !(pCtx->cr0 & X86_CR0_PE)
4053 || pCtx->cs.Attr.n.u4Type == 3)
4054 {
4055 Assert(!pCtx->ss.Attr.n.u2Dpl);
4056 }
4057 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4058 {
4059 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4060 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4061 Assert(pCtx->ss.Attr.n.u1Present);
4062 Assert(!(pCtx->ss.Attr.u & 0xf00));
4063 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4064 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4065 || !(pCtx->ss.Attr.n.u1Granularity));
4066 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4067 || (pCtx->ss.Attr.n.u1Granularity));
4068 }
4069 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4070 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4071 {
4072 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4073 Assert(pCtx->ds.Attr.n.u1Present);
4074 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4075 Assert(!(pCtx->ds.Attr.u & 0xf00));
4076 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4077 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4078 || !(pCtx->ds.Attr.n.u1Granularity));
4079 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4080 || (pCtx->ds.Attr.n.u1Granularity));
4081 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4082 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4083 }
4084 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4085 {
4086 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4087 Assert(pCtx->es.Attr.n.u1Present);
4088 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4089 Assert(!(pCtx->es.Attr.u & 0xf00));
4090 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4091 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4092 || !(pCtx->es.Attr.n.u1Granularity));
4093 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4094 || (pCtx->es.Attr.n.u1Granularity));
4095 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4096 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4097 }
4098 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4099 {
4100 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4101 Assert(pCtx->fs.Attr.n.u1Present);
4102 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4103 Assert(!(pCtx->fs.Attr.u & 0xf00));
4104 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4105 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4106 || !(pCtx->fs.Attr.n.u1Granularity));
4107 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4108 || (pCtx->fs.Attr.n.u1Granularity));
4109 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4110 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4111 }
4112 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4113 {
4114 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4115 Assert(pCtx->gs.Attr.n.u1Present);
4116 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4117 Assert(!(pCtx->gs.Attr.u & 0xf00));
4118 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4119 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4120 || !(pCtx->gs.Attr.n.u1Granularity));
4121 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4122 || (pCtx->gs.Attr.n.u1Granularity));
4123 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4124 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4125 }
4126 /* 64-bit capable CPUs. */
4127# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4128 if (HMVMX_IS_64BIT_HOST_MODE())
4129 {
4130 Assert(!(pCtx->cs.u64Base >> 32));
4131 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4132 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4133 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4134 }
4135# endif
4136 }
4137 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4138 || ( CPUMIsGuestInRealModeEx(pCtx)
4139 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4140 {
4141 /* Real and v86 mode checks. */
4142 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4143 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4144 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4145 {
4146 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4147 }
4148 else
4149 {
4150 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4151 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4152 }
4153
4154 /* CS */
4155 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4156 Assert(pCtx->cs.u32Limit == 0xffff);
4157 Assert(u32CSAttr == 0xf3);
4158 /* SS */
4159 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4160 Assert(pCtx->ss.u32Limit == 0xffff);
4161 Assert(u32SSAttr == 0xf3);
4162 /* DS */
4163 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4164 Assert(pCtx->ds.u32Limit == 0xffff);
4165 Assert(u32DSAttr == 0xf3);
4166 /* ES */
4167 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4168 Assert(pCtx->es.u32Limit == 0xffff);
4169 Assert(u32ESAttr == 0xf3);
4170 /* FS */
4171 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4172 Assert(pCtx->fs.u32Limit == 0xffff);
4173 Assert(u32FSAttr == 0xf3);
4174 /* GS */
4175 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4176 Assert(pCtx->gs.u32Limit == 0xffff);
4177 Assert(u32GSAttr == 0xf3);
4178 /* 64-bit capable CPUs. */
4179# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4180 if (HMVMX_IS_64BIT_HOST_MODE())
4181 {
4182 Assert(!(pCtx->cs.u64Base >> 32));
4183 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4184 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4185 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4186 }
4187# endif
4188 }
4189}
4190#endif /* VBOX_STRICT */
4191
4192
4193/**
4194 * Writes a guest segment register into the guest-state area in the VMCS.
4195 *
4196 * @returns VBox status code.
4197 * @param pVCpu Pointer to the VMCPU.
4198 * @param idxSel Index of the selector in the VMCS.
4199 * @param idxLimit Index of the segment limit in the VMCS.
4200 * @param idxBase Index of the segment base in the VMCS.
4201 * @param idxAccess Index of the access rights of the segment in the VMCS.
4202 * @param pSelReg Pointer to the segment selector.
4203 *
4204 * @remarks No-long-jump zone!!!
4205 */
4206static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4207 uint32_t idxAccess, PCPUMSELREG pSelReg)
4208{
4209 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4210 AssertRCReturn(rc, rc);
4211 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4212 AssertRCReturn(rc, rc);
4213 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4214 AssertRCReturn(rc, rc);
4215
4216 uint32_t u32Access = pSelReg->Attr.u;
4217 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4218 {
4219 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4220 u32Access = 0xf3;
4221 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4222 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4223 }
4224 else
4225 {
4226 /*
4227 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4228 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4229 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4230 * loaded in protected-mode have their attribute as 0.
4231 */
4232 if (!u32Access)
4233 u32Access = X86DESCATTR_UNUSABLE;
4234 }
4235
4236 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4237 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4238 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4239
4240 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4241 AssertRCReturn(rc, rc);
4242 return rc;
4243}
4244
4245
4246/**
4247 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4248 * into the guest-state area in the VMCS.
4249 *
4250 * @returns VBox status code.
4251 * @param pVM Pointer to the VM.
4252 * @param pVCPU Pointer to the VMCPU.
4253 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4254 * out-of-sync. Make sure to update the required fields
4255 * before using them.
4256 *
4257 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4258 * @remarks No-long-jump zone!!!
4259 */
4260static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4261{
4262 int rc = VERR_INTERNAL_ERROR_5;
4263 PVM pVM = pVCpu->CTX_SUFF(pVM);
4264
4265 /*
4266 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4267 */
4268 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4269 {
4270 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4271 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4272 {
4273 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4274 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4275 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4276 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4277 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4278 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4279 }
4280
4281#ifdef VBOX_WITH_REM
4282 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4283 {
4284 Assert(pVM->hm.s.vmx.pRealModeTSS);
4285 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4286 if ( pVCpu->hm.s.vmx.fWasInRealMode
4287 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4288 {
4289 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4290 in real-mode (e.g. OpenBSD 4.0) */
4291 REMFlushTBs(pVM);
4292 Log4(("Load: Switch to protected mode detected!\n"));
4293 pVCpu->hm.s.vmx.fWasInRealMode = false;
4294 }
4295 }
4296#endif
4297 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4298 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4299 AssertRCReturn(rc, rc);
4300 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4301 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4302 AssertRCReturn(rc, rc);
4303 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4304 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4305 AssertRCReturn(rc, rc);
4306 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4307 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4308 AssertRCReturn(rc, rc);
4309 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4310 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4311 AssertRCReturn(rc, rc);
4312 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4313 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4314 AssertRCReturn(rc, rc);
4315
4316#ifdef VBOX_STRICT
4317 /* Validate. */
4318 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4319#endif
4320
4321 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4322 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4323 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4324 }
4325
4326 /*
4327 * Guest TR.
4328 */
4329 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4330 {
4331 /*
4332 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4333 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4334 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4335 */
4336 uint16_t u16Sel = 0;
4337 uint32_t u32Limit = 0;
4338 uint64_t u64Base = 0;
4339 uint32_t u32AccessRights = 0;
4340
4341 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4342 {
4343 u16Sel = pMixedCtx->tr.Sel;
4344 u32Limit = pMixedCtx->tr.u32Limit;
4345 u64Base = pMixedCtx->tr.u64Base;
4346 u32AccessRights = pMixedCtx->tr.Attr.u;
4347 }
4348 else
4349 {
4350 Assert(pVM->hm.s.vmx.pRealModeTSS);
4351 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4352
4353 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4354 RTGCPHYS GCPhys;
4355 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4356 AssertRCReturn(rc, rc);
4357
4358 X86DESCATTR DescAttr;
4359 DescAttr.u = 0;
4360 DescAttr.n.u1Present = 1;
4361 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4362
4363 u16Sel = 0;
4364 u32Limit = HM_VTX_TSS_SIZE;
4365 u64Base = GCPhys; /* in real-mode phys = virt. */
4366 u32AccessRights = DescAttr.u;
4367 }
4368
4369 /* Validate. */
4370 Assert(!(u16Sel & RT_BIT(2)));
4371 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4372 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4373 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4374 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4375 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4376 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4377 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4378 Assert( (u32Limit & 0xfff) == 0xfff
4379 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4380 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4381 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4382
4383 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4384 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4385 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4386 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4387
4388 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4389 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4390 }
4391
4392 /*
4393 * Guest GDTR.
4394 */
4395 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4396 {
4397 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4398 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4399
4400 /* Validate. */
4401 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4402
4403 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4404 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4405 }
4406
4407 /*
4408 * Guest LDTR.
4409 */
4410 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4411 {
4412 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4413 uint32_t u32Access = 0;
4414 if (!pMixedCtx->ldtr.Attr.u)
4415 u32Access = X86DESCATTR_UNUSABLE;
4416 else
4417 u32Access = pMixedCtx->ldtr.Attr.u;
4418
4419 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4420 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4421 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4422 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4423
4424 /* Validate. */
4425 if (!(u32Access & X86DESCATTR_UNUSABLE))
4426 {
4427 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4428 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4429 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4430 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4431 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4432 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4433 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4434 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4435 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4436 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4437 }
4438
4439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4440 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4441 }
4442
4443 /*
4444 * Guest IDTR.
4445 */
4446 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4447 {
4448 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4449 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4450
4451 /* Validate. */
4452 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4453
4454 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4455 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4456 }
4457
4458 return VINF_SUCCESS;
4459}
4460
4461
4462/**
4463 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4464 * areas. These MSRs will automatically be loaded to the host CPU on every
4465 * successful VM entry and stored from the host CPU on every successful VM-exit.
4466 *
4467 * This also creates/updates MSR slots for the host MSRs. The actual host
4468 * MSR values are -not- updated here for performance reasons. See
4469 * hmR0VmxSaveHostMsrs().
4470 *
4471 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu Pointer to the VMCPU.
4475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4476 * out-of-sync. Make sure to update the required fields
4477 * before using them.
4478 *
4479 * @remarks No-long-jump zone!!!
4480 */
4481static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4482{
4483 AssertPtr(pVCpu);
4484 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4485
4486 /*
4487 * MSRs that we use the auto-load/store MSR area in the VMCS.
4488 */
4489 PVM pVM = pVCpu->CTX_SUFF(pVM);
4490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4491 {
4492#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4493 if (pVM->hm.s.fAllow64BitGuests)
4494 {
4495 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4496 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4497 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4498 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4499# ifdef DEBUG
4500 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4501 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4502 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4503# endif
4504 }
4505#endif
4506 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4507 }
4508
4509 /*
4510 * Guest Sysenter MSRs.
4511 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4512 * VM-exits on WRMSRs for these MSRs.
4513 */
4514 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4515 {
4516 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4517 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4518 }
4519
4520 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4521 {
4522 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4523 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4524 }
4525
4526 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4527 {
4528 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4529 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4530 }
4531
4532 return VINF_SUCCESS;
4533}
4534
4535
4536/**
4537 * Loads the guest activity state into the guest-state area in the VMCS.
4538 *
4539 * @returns VBox status code.
4540 * @param pVCpu Pointer to the VMCPU.
4541 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4542 * out-of-sync. Make sure to update the required fields
4543 * before using them.
4544 *
4545 * @remarks No-long-jump zone!!!
4546 */
4547static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4548{
4549 NOREF(pCtx);
4550 /** @todo See if we can make use of other states, e.g.
4551 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4552 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4553 {
4554 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4555 AssertRCReturn(rc, rc);
4556
4557 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4558 }
4559 return VINF_SUCCESS;
4560}
4561
4562
4563/**
4564 * Sets up the appropriate function to run guest code.
4565 *
4566 * @returns VBox status code.
4567 * @param pVCpu Pointer to the VMCPU.
4568 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4569 * out-of-sync. Make sure to update the required fields
4570 * before using them.
4571 *
4572 * @remarks No-long-jump zone!!!
4573 */
4574static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4575{
4576 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4577 {
4578#ifndef VBOX_ENABLE_64_BITS_GUESTS
4579 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4580#endif
4581 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4582#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4583 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4584 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4585 {
4586 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4587 {
4588 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4589 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS),
4590 ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4591 }
4592 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4593 }
4594#else
4595 /* 64-bit host or hybrid host. */
4596 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4597#endif
4598 }
4599 else
4600 {
4601 /* Guest is not in long mode, use the 32-bit handler. */
4602#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4603 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4604 {
4605 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4606 {
4607 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4608 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS),
4609 ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4610 }
4611 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4612 }
4613#else
4614 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4615#endif
4616 }
4617 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4618 return VINF_SUCCESS;
4619}
4620
4621
4622/**
4623 * Wrapper for running the guest code in VT-x.
4624 *
4625 * @returns VBox strict status code.
4626 * @param pVM Pointer to the VM.
4627 * @param pVCpu Pointer to the VMCPU.
4628 * @param pCtx Pointer to the guest-CPU context.
4629 *
4630 * @remarks No-long-jump zone!!!
4631 */
4632DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4633{
4634 /*
4635 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4636 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4637 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4638 */
4639 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4640 /** @todo Add stats for resume vs launch. */
4641#ifdef VBOX_WITH_KERNEL_USING_XMM
4642 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4643#else
4644 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4645#endif
4646}
4647
4648
4649/**
4650 * Reports world-switch error and dumps some useful debug info.
4651 *
4652 * @param pVM Pointer to the VM.
4653 * @param pVCpu Pointer to the VMCPU.
4654 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4655 * @param pCtx Pointer to the guest-CPU context.
4656 * @param pVmxTransient Pointer to the VMX transient structure (only
4657 * exitReason updated).
4658 */
4659static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4660{
4661 Assert(pVM);
4662 Assert(pVCpu);
4663 Assert(pCtx);
4664 Assert(pVmxTransient);
4665 HMVMX_ASSERT_PREEMPT_SAFE();
4666
4667 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4668 switch (rcVMRun)
4669 {
4670 case VERR_VMX_INVALID_VMXON_PTR:
4671 AssertFailed();
4672 break;
4673 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4674 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4675 {
4676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4677 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4678 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4679 AssertRC(rc);
4680
4681 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4682 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4683 Cannot do it here as we may have been long preempted. */
4684
4685#ifdef VBOX_STRICT
4686 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4687 pVmxTransient->uExitReason));
4688 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4689 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4690 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4691 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4692 else
4693 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4694 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4695 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4696
4697 /* VMX control bits. */
4698 uint32_t u32Val;
4699 uint64_t u64Val;
4700 HMVMXHCUINTREG uHCReg;
4701 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4702 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4703 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4704 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4705 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4706 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4707 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4708 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4709 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4710 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4712 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4713 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4714 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4715 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4716 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4718 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4719 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4720 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4721 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4722 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4723 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4724 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4725 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4726 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4727 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4728 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4729 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4730 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4731 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4732 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4733 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4734 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4735 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4736 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4737 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4738 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4739 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4740 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4741 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4742 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4743
4744 /* Guest bits. */
4745 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4746 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4747 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4748 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4749 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4750 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4751 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4752 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4753
4754 /* Host bits. */
4755 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4756 Log4(("Host CR0 %#RHr\n", uHCReg));
4757 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4758 Log4(("Host CR3 %#RHr\n", uHCReg));
4759 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4760 Log4(("Host CR4 %#RHr\n", uHCReg));
4761
4762 RTGDTR HostGdtr;
4763 PCX86DESCHC pDesc;
4764 ASMGetGDTR(&HostGdtr);
4765 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4766 Log4(("Host CS %#08x\n", u32Val));
4767 if (u32Val < HostGdtr.cbGdt)
4768 {
4769 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4770 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4771 }
4772
4773 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4774 Log4(("Host DS %#08x\n", u32Val));
4775 if (u32Val < HostGdtr.cbGdt)
4776 {
4777 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4778 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4779 }
4780
4781 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4782 Log4(("Host ES %#08x\n", u32Val));
4783 if (u32Val < HostGdtr.cbGdt)
4784 {
4785 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4786 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4787 }
4788
4789 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4790 Log4(("Host FS %#08x\n", u32Val));
4791 if (u32Val < HostGdtr.cbGdt)
4792 {
4793 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4794 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4795 }
4796
4797 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4798 Log4(("Host GS %#08x\n", u32Val));
4799 if (u32Val < HostGdtr.cbGdt)
4800 {
4801 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4802 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4803 }
4804
4805 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4806 Log4(("Host SS %#08x\n", u32Val));
4807 if (u32Val < HostGdtr.cbGdt)
4808 {
4809 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4810 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4811 }
4812
4813 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4814 Log4(("Host TR %#08x\n", u32Val));
4815 if (u32Val < HostGdtr.cbGdt)
4816 {
4817 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4818 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4819 }
4820
4821 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4822 Log4(("Host TR Base %#RHv\n", uHCReg));
4823 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4824 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4825 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4826 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4827 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4828 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4829 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4830 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4831 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4832 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4833 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4834 Log4(("Host RSP %#RHv\n", uHCReg));
4835 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4836 Log4(("Host RIP %#RHv\n", uHCReg));
4837# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4838 if (HMVMX_IS_64BIT_HOST_MODE())
4839 {
4840 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4841 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4842 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4843 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4844 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4845 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4846 }
4847# endif
4848#endif /* VBOX_STRICT */
4849 break;
4850 }
4851
4852 default:
4853 /* Impossible */
4854 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4855 break;
4856 }
4857 NOREF(pVM); NOREF(pCtx);
4858}
4859
4860
4861#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4862#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4863# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4864#endif
4865#ifdef VBOX_STRICT
4866static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4867{
4868 switch (idxField)
4869 {
4870 case VMX_VMCS_GUEST_RIP:
4871 case VMX_VMCS_GUEST_RSP:
4872 case VMX_VMCS_GUEST_SYSENTER_EIP:
4873 case VMX_VMCS_GUEST_SYSENTER_ESP:
4874 case VMX_VMCS_GUEST_GDTR_BASE:
4875 case VMX_VMCS_GUEST_IDTR_BASE:
4876 case VMX_VMCS_GUEST_CS_BASE:
4877 case VMX_VMCS_GUEST_DS_BASE:
4878 case VMX_VMCS_GUEST_ES_BASE:
4879 case VMX_VMCS_GUEST_FS_BASE:
4880 case VMX_VMCS_GUEST_GS_BASE:
4881 case VMX_VMCS_GUEST_SS_BASE:
4882 case VMX_VMCS_GUEST_LDTR_BASE:
4883 case VMX_VMCS_GUEST_TR_BASE:
4884 case VMX_VMCS_GUEST_CR3:
4885 return true;
4886 }
4887 return false;
4888}
4889
4890static bool hmR0VmxIsValidReadField(uint32_t idxField)
4891{
4892 switch (idxField)
4893 {
4894 /* Read-only fields. */
4895 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4896 return true;
4897 }
4898 /* Remaining readable fields should also be writable. */
4899 return hmR0VmxIsValidWriteField(idxField);
4900}
4901#endif /* VBOX_STRICT */
4902
4903
4904/**
4905 * Executes the specified handler in 64-bit mode.
4906 *
4907 * @returns VBox status code.
4908 * @param pVM Pointer to the VM.
4909 * @param pVCpu Pointer to the VMCPU.
4910 * @param pCtx Pointer to the guest CPU context.
4911 * @param enmOp The operation to perform.
4912 * @param cbParam Number of parameters.
4913 * @param paParam Array of 32-bit parameters.
4914 */
4915VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4916 uint32_t *paParam)
4917{
4918 int rc, rc2;
4919 PHMGLOBALCPUINFO pCpu;
4920 RTHCPHYS HCPhysCpuPage;
4921 RTCCUINTREG uOldEflags;
4922
4923 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4924 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4925 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4926 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4927
4928#ifdef VBOX_STRICT
4929 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4930 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4931
4932 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4933 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4934#endif
4935
4936 /* Disable interrupts. */
4937 uOldEflags = ASMIntDisableFlags();
4938
4939#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4940 RTCPUID idHostCpu = RTMpCpuId();
4941 CPUMR0SetLApic(pVCpu, idHostCpu);
4942#endif
4943
4944 pCpu = HMR0GetCurrentCpu();
4945 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4946
4947 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4948 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4949
4950 /* Leave VMX Root Mode. */
4951 VMXDisable();
4952
4953 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4954
4955 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4956 CPUMSetHyperEIP(pVCpu, enmOp);
4957 for (int i = (int)cbParam - 1; i >= 0; i--)
4958 CPUMPushHyper(pVCpu, paParam[i]);
4959
4960 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4961
4962 /* Call the switcher. */
4963 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4964 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4965
4966 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4967 /* Make sure the VMX instructions don't cause #UD faults. */
4968 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4969
4970 /* Re-enter VMX Root Mode */
4971 rc2 = VMXEnable(HCPhysCpuPage);
4972 if (RT_FAILURE(rc2))
4973 {
4974 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4975 ASMSetFlags(uOldEflags);
4976 return rc2;
4977 }
4978
4979 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4980 AssertRC(rc2);
4981 Assert(!(ASMGetFlags() & X86_EFL_IF));
4982 ASMSetFlags(uOldEflags);
4983 return rc;
4984}
4985
4986
4987/**
4988 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4989 * supporting 64-bit guests.
4990 *
4991 * @returns VBox status code.
4992 * @param fResume Whether to VMLAUNCH or VMRESUME.
4993 * @param pCtx Pointer to the guest-CPU context.
4994 * @param pCache Pointer to the VMCS cache.
4995 * @param pVM Pointer to the VM.
4996 * @param pVCpu Pointer to the VMCPU.
4997 */
4998DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4999{
5000 uint32_t aParam[6];
5001 PHMGLOBALCPUINFO pCpu = NULL;
5002 RTHCPHYS HCPhysCpuPage = 0;
5003 int rc = VERR_INTERNAL_ERROR_5;
5004
5005 pCpu = HMR0GetCurrentCpu();
5006 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5007
5008#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5009 pCache->uPos = 1;
5010 pCache->interPD = PGMGetInterPaeCR3(pVM);
5011 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5012#endif
5013
5014#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5015 pCache->TestIn.HCPhysCpuPage = 0;
5016 pCache->TestIn.HCPhysVmcs = 0;
5017 pCache->TestIn.pCache = 0;
5018 pCache->TestOut.HCPhysVmcs = 0;
5019 pCache->TestOut.pCache = 0;
5020 pCache->TestOut.pCtx = 0;
5021 pCache->TestOut.eflags = 0;
5022#endif
5023
5024 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5025 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5026 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5027 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5028 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5029 aParam[5] = 0;
5030
5031#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5032 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5033 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5034#endif
5035 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5036
5037#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5038 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5039 Assert(pCtx->dr[4] == 10);
5040 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5041#endif
5042
5043#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5044 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5045 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5046 pVCpu->hm.s.vmx.HCPhysVmcs));
5047 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5048 pCache->TestOut.HCPhysVmcs));
5049 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5050 pCache->TestOut.pCache));
5051 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5052 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5053 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5054 pCache->TestOut.pCtx));
5055 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5056#endif
5057 return rc;
5058}
5059
5060
5061/**
5062 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5063 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5064 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5065 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5066 *
5067 * @returns VBox status code.
5068 * @param pVM Pointer to the VM.
5069 * @param pVCpu Pointer to the VMCPU.
5070 */
5071static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5072{
5073#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5074{ \
5075 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5076 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5077 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5078 ++cReadFields; \
5079}
5080
5081 AssertPtr(pVM);
5082 AssertPtr(pVCpu);
5083 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5084 uint32_t cReadFields = 0;
5085
5086 /*
5087 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5088 * and serve to indicate exceptions to the rules.
5089 */
5090
5091 /* Guest-natural selector base fields. */
5092#if 0
5093 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5094 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5095 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5096#endif
5097 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5098 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5099 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5100 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5101 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5102 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5103 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5104 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5105 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5106 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5107 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5108 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5109#if 0
5110 /* Unused natural width guest-state fields. */
5111 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5112 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5113#endif
5114 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5115 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5116
5117 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5118#if 0
5119 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5120 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5121 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5122 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5123 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5124 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5125 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5126 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5127 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5128#endif
5129
5130 /* Natural width guest-state fields. */
5131 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5132#if 0
5133 /* Currently unused field. */
5134 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5135#endif
5136
5137 if (pVM->hm.s.fNestedPaging)
5138 {
5139 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5140 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5141 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5142 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5143 }
5144 else
5145 {
5146 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5147 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5148 }
5149
5150#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5151 return VINF_SUCCESS;
5152}
5153
5154
5155/**
5156 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5157 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5158 * darwin, running 64-bit guests).
5159 *
5160 * @returns VBox status code.
5161 * @param pVCpu Pointer to the VMCPU.
5162 * @param idxField The VMCS field encoding.
5163 * @param u64Val 16, 32 or 64-bit value.
5164 */
5165VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5166{
5167 int rc;
5168 switch (idxField)
5169 {
5170 /*
5171 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5172 */
5173 /* 64-bit Control fields. */
5174 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5175 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5176 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5177 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5178 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5179 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5180 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5181 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5182 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5183 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5184 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5185 case VMX_VMCS64_CTRL_EPTP_FULL:
5186 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5187 /* 64-bit Guest-state fields. */
5188 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5189 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5190 case VMX_VMCS64_GUEST_PAT_FULL:
5191 case VMX_VMCS64_GUEST_EFER_FULL:
5192 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5193 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5194 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5195 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5196 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5197 /* 64-bit Host-state fields. */
5198 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5199 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5200 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5201 {
5202 rc = VMXWriteVmcs32(idxField, u64Val);
5203 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5204 break;
5205 }
5206
5207 /*
5208 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5209 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5210 */
5211 /* Natural-width Guest-state fields. */
5212 case VMX_VMCS_GUEST_CR3:
5213 case VMX_VMCS_GUEST_ES_BASE:
5214 case VMX_VMCS_GUEST_CS_BASE:
5215 case VMX_VMCS_GUEST_SS_BASE:
5216 case VMX_VMCS_GUEST_DS_BASE:
5217 case VMX_VMCS_GUEST_FS_BASE:
5218 case VMX_VMCS_GUEST_GS_BASE:
5219 case VMX_VMCS_GUEST_LDTR_BASE:
5220 case VMX_VMCS_GUEST_TR_BASE:
5221 case VMX_VMCS_GUEST_GDTR_BASE:
5222 case VMX_VMCS_GUEST_IDTR_BASE:
5223 case VMX_VMCS_GUEST_RSP:
5224 case VMX_VMCS_GUEST_RIP:
5225 case VMX_VMCS_GUEST_SYSENTER_ESP:
5226 case VMX_VMCS_GUEST_SYSENTER_EIP:
5227 {
5228 if (!(u64Val >> 32))
5229 {
5230 /* If this field is 64-bit, VT-x will zero out the top bits. */
5231 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5232 }
5233 else
5234 {
5235 /* Assert that only the 32->64 switcher case should ever come here. */
5236 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5237 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5238 }
5239 break;
5240 }
5241
5242 default:
5243 {
5244 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5245 rc = VERR_INVALID_PARAMETER;
5246 break;
5247 }
5248 }
5249 AssertRCReturn(rc, rc);
5250 return rc;
5251}
5252
5253
5254/**
5255 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5256 * hosts (except darwin) for 64-bit guests.
5257 *
5258 * @param pVCpu Pointer to the VMCPU.
5259 * @param idxField The VMCS field encoding.
5260 * @param u64Val 16, 32 or 64-bit value.
5261 */
5262VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5263{
5264 AssertPtr(pVCpu);
5265 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5266
5267 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5268 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5269
5270 /* Make sure there are no duplicates. */
5271 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5272 {
5273 if (pCache->Write.aField[i] == idxField)
5274 {
5275 pCache->Write.aFieldVal[i] = u64Val;
5276 return VINF_SUCCESS;
5277 }
5278 }
5279
5280 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5281 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5282 pCache->Write.cValidEntries++;
5283 return VINF_SUCCESS;
5284}
5285
5286/* Enable later when the assembly code uses these as callbacks. */
5287#if 0
5288/*
5289 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5290 *
5291 * @param pVCpu Pointer to the VMCPU.
5292 * @param pCache Pointer to the VMCS cache.
5293 *
5294 * @remarks No-long-jump zone!!!
5295 */
5296VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5297{
5298 AssertPtr(pCache);
5299 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5300 {
5301 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5302 AssertRC(rc);
5303 }
5304 pCache->Write.cValidEntries = 0;
5305}
5306
5307
5308/**
5309 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5310 *
5311 * @param pVCpu Pointer to the VMCPU.
5312 * @param pCache Pointer to the VMCS cache.
5313 *
5314 * @remarks No-long-jump zone!!!
5315 */
5316VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5317{
5318 AssertPtr(pCache);
5319 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5320 {
5321 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5322 AssertRC(rc);
5323 }
5324}
5325#endif
5326#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5327
5328
5329/**
5330 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5331 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5332 * timer.
5333 *
5334 * @returns VBox status code.
5335 * @param pVCpu Pointer to the VMCPU.
5336 *
5337 * @remarks No-long-jump zone!!!
5338 */
5339static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5340{
5341 int rc = VERR_INTERNAL_ERROR_5;
5342 bool fOffsettedTsc = false;
5343 PVM pVM = pVCpu->CTX_SUFF(pVM);
5344 if (pVM->hm.s.vmx.fUsePreemptTimer)
5345 {
5346 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5347
5348 /* Make sure the returned values have sane upper and lower boundaries. */
5349 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5350 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5351 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5352 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5353
5354 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5355 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5356 }
5357 else
5358 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5359
5360 if (fOffsettedTsc)
5361 {
5362 uint64_t u64CurTSC = ASMReadTSC();
5363 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5364 {
5365 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5366 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5367
5368 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5369 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5370 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5371 }
5372 else
5373 {
5374 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5375 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5376 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5377 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5378 }
5379 }
5380 else
5381 {
5382 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5383 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5384 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5385 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5386 }
5387}
5388
5389
5390/**
5391 * Determines if an exception is a contributory exception. Contributory
5392 * exceptions are ones which can cause double-faults. Page-fault is
5393 * intentionally not included here as it's a conditional contributory exception.
5394 *
5395 * @returns true if the exception is contributory, false otherwise.
5396 * @param uVector The exception vector.
5397 */
5398DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5399{
5400 switch (uVector)
5401 {
5402 case X86_XCPT_GP:
5403 case X86_XCPT_SS:
5404 case X86_XCPT_NP:
5405 case X86_XCPT_TS:
5406 case X86_XCPT_DE:
5407 return true;
5408 default:
5409 break;
5410 }
5411 return false;
5412}
5413
5414
5415/**
5416 * Sets an event as a pending event to be injected into the guest.
5417 *
5418 * @param pVCpu Pointer to the VMCPU.
5419 * @param u32IntInfo The VM-entry interruption-information field.
5420 * @param cbInstr The VM-entry instruction length in bytes (for software
5421 * interrupts, exceptions and privileged software
5422 * exceptions).
5423 * @param u32ErrCode The VM-entry exception error code.
5424 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5425 * page-fault.
5426 *
5427 * @remarks Statistics counter assumes this is a guest event being injected or
5428 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5429 * always incremented.
5430 */
5431DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5432 RTGCUINTPTR GCPtrFaultAddress)
5433{
5434 Assert(!pVCpu->hm.s.Event.fPending);
5435 pVCpu->hm.s.Event.fPending = true;
5436 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5437 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5438 pVCpu->hm.s.Event.cbInstr = cbInstr;
5439 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5440
5441 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5442}
5443
5444
5445/**
5446 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5447 *
5448 * @param pVCpu Pointer to the VMCPU.
5449 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5450 * out-of-sync. Make sure to update the required fields
5451 * before using them.
5452 */
5453DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5454{
5455 NOREF(pMixedCtx);
5456 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5457 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5458 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5459 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5460}
5461
5462
5463/**
5464 * Handle a condition that occurred while delivering an event through the guest
5465 * IDT.
5466 *
5467 * @returns VBox status code (informational error codes included).
5468 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5469 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5470 * continue execution of the guest which will delivery the #DF.
5471 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5472 *
5473 * @param pVCpu Pointer to the VMCPU.
5474 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5475 * out-of-sync. Make sure to update the required fields
5476 * before using them.
5477 * @param pVmxTransient Pointer to the VMX transient structure.
5478 *
5479 * @remarks No-long-jump zone!!!
5480 */
5481static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5482{
5483 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5484 AssertRCReturn(rc, rc);
5485 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5486 {
5487 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5488 AssertRCReturn(rc, rc);
5489
5490 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5491 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5492 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5493
5494 typedef enum
5495 {
5496 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5497 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5498 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5499 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5500 } VMXREFLECTXCPT;
5501
5502 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5503 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5504 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5505 {
5506 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5507 {
5508 enmReflect = VMXREFLECTXCPT_XCPT;
5509#ifdef VBOX_STRICT
5510 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5511 && uExitVector == X86_XCPT_PF)
5512 {
5513 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5514 }
5515#endif
5516 if ( uExitVector == X86_XCPT_PF
5517 && uIdtVector == X86_XCPT_PF)
5518 {
5519 pVmxTransient->fVectoringPF = true;
5520 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5521 }
5522 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5523 && hmR0VmxIsContributoryXcpt(uExitVector)
5524 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5525 || uIdtVector == X86_XCPT_PF))
5526 {
5527 enmReflect = VMXREFLECTXCPT_DF;
5528 }
5529 else if (uIdtVector == X86_XCPT_DF)
5530 enmReflect = VMXREFLECTXCPT_TF;
5531 }
5532 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5533 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5534 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5535 {
5536 /*
5537 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5538 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5539 */
5540 enmReflect = VMXREFLECTXCPT_XCPT;
5541 }
5542 }
5543 else
5544 {
5545 /*
5546 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5547 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5548 * original exception to the guest after handling the VM-exit.
5549 */
5550 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5551 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5552 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5553 {
5554 enmReflect = VMXREFLECTXCPT_XCPT;
5555 }
5556 }
5557
5558 switch (enmReflect)
5559 {
5560 case VMXREFLECTXCPT_XCPT:
5561 {
5562 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5563 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5564 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5565
5566 uint32_t u32ErrCode = 0;
5567 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5568 {
5569 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5570 AssertRCReturn(rc, rc);
5571 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5572 }
5573
5574 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5575 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5576 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5577 rc = VINF_SUCCESS;
5578 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5579 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5580
5581 break;
5582 }
5583
5584 case VMXREFLECTXCPT_DF:
5585 {
5586 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5587 rc = VINF_HM_DOUBLE_FAULT;
5588 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5589 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5590
5591 break;
5592 }
5593
5594 case VMXREFLECTXCPT_TF:
5595 {
5596 rc = VINF_EM_RESET;
5597 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5598 uExitVector));
5599 break;
5600 }
5601
5602 default:
5603 Assert(rc == VINF_SUCCESS);
5604 break;
5605 }
5606 }
5607 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5608 return rc;
5609}
5610
5611
5612/**
5613 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5614 *
5615 * @returns VBox status code.
5616 * @param pVCpu Pointer to the VMCPU.
5617 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5618 * out-of-sync. Make sure to update the required fields
5619 * before using them.
5620 *
5621 * @remarks No-long-jump zone!!!
5622 */
5623static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5624{
5625 NOREF(pMixedCtx);
5626
5627 /*
5628 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5629 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5630 */
5631 VMMRZCallRing3Disable(pVCpu);
5632 HM_DISABLE_PREEMPT_IF_NEEDED();
5633
5634 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5635 {
5636 uint32_t uVal = 0;
5637 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5638 AssertRCReturn(rc, rc);
5639
5640 uint32_t uShadow = 0;
5641 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5642 AssertRCReturn(rc, rc);
5643
5644 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5645 CPUMSetGuestCR0(pVCpu, uVal);
5646 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5647 }
5648
5649 HM_RESTORE_PREEMPT_IF_NEEDED();
5650 VMMRZCallRing3Enable(pVCpu);
5651 return VINF_SUCCESS;
5652}
5653
5654
5655/**
5656 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5657 *
5658 * @returns VBox status code.
5659 * @param pVCpu Pointer to the VMCPU.
5660 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5661 * out-of-sync. Make sure to update the required fields
5662 * before using them.
5663 *
5664 * @remarks No-long-jump zone!!!
5665 */
5666static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5667{
5668 NOREF(pMixedCtx);
5669
5670 int rc = VINF_SUCCESS;
5671 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5672 {
5673 uint32_t uVal = 0;
5674 uint32_t uShadow = 0;
5675 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5676 AssertRCReturn(rc, rc);
5677 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5678 AssertRCReturn(rc, rc);
5679
5680 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5681 CPUMSetGuestCR4(pVCpu, uVal);
5682 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5683 }
5684 return rc;
5685}
5686
5687
5688/**
5689 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5690 *
5691 * @returns VBox status code.
5692 * @param pVCpu Pointer to the VMCPU.
5693 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5694 * out-of-sync. Make sure to update the required fields
5695 * before using them.
5696 *
5697 * @remarks No-long-jump zone!!!
5698 */
5699static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5700{
5701 int rc = VINF_SUCCESS;
5702 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5703 {
5704 uint64_t u64Val = 0;
5705 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5706 AssertRCReturn(rc, rc);
5707
5708 pMixedCtx->rip = u64Val;
5709 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5710 }
5711 return rc;
5712}
5713
5714
5715/**
5716 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5717 *
5718 * @returns VBox status code.
5719 * @param pVCpu Pointer to the VMCPU.
5720 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5721 * out-of-sync. Make sure to update the required fields
5722 * before using them.
5723 *
5724 * @remarks No-long-jump zone!!!
5725 */
5726static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5727{
5728 int rc = VINF_SUCCESS;
5729 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5730 {
5731 uint64_t u64Val = 0;
5732 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5733 AssertRCReturn(rc, rc);
5734
5735 pMixedCtx->rsp = u64Val;
5736 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5737 }
5738 return rc;
5739}
5740
5741
5742/**
5743 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5744 *
5745 * @returns VBox status code.
5746 * @param pVCpu Pointer to the VMCPU.
5747 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5748 * out-of-sync. Make sure to update the required fields
5749 * before using them.
5750 *
5751 * @remarks No-long-jump zone!!!
5752 */
5753static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5754{
5755 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5756 {
5757 uint32_t uVal = 0;
5758 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5759 AssertRCReturn(rc, rc);
5760
5761 pMixedCtx->eflags.u32 = uVal;
5762 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5763 {
5764 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5765 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5766
5767 pMixedCtx->eflags.Bits.u1VM = 0;
5768 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5769 }
5770
5771 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5772 }
5773 return VINF_SUCCESS;
5774}
5775
5776
5777/**
5778 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5779 * guest-CPU context.
5780 */
5781DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5782{
5783 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5784 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5785 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5786 return rc;
5787}
5788
5789
5790/**
5791 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5792 * from the guest-state area in the VMCS.
5793 *
5794 * @param pVCpu Pointer to the VMCPU.
5795 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5796 * out-of-sync. Make sure to update the required fields
5797 * before using them.
5798 *
5799 * @remarks No-long-jump zone!!!
5800 */
5801static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5802{
5803 uint32_t uIntrState = 0;
5804 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5805 AssertRC(rc);
5806
5807 if (!uIntrState)
5808 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5809 else
5810 {
5811 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5812 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5813 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5814 AssertRC(rc);
5815 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5816 AssertRC(rc);
5817
5818 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5819 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5820 }
5821}
5822
5823
5824/**
5825 * Saves the guest's activity state.
5826 *
5827 * @returns VBox status code.
5828 * @param pVCpu Pointer to the VMCPU.
5829 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5830 * out-of-sync. Make sure to update the required fields
5831 * before using them.
5832 *
5833 * @remarks No-long-jump zone!!!
5834 */
5835static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5836{
5837 NOREF(pMixedCtx);
5838 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5839 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5840 return VINF_SUCCESS;
5841}
5842
5843
5844/**
5845 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5846 * the current VMCS into the guest-CPU context.
5847 *
5848 * @returns VBox status code.
5849 * @param pVCpu Pointer to the VMCPU.
5850 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5851 * out-of-sync. Make sure to update the required fields
5852 * before using them.
5853 *
5854 * @remarks No-long-jump zone!!!
5855 */
5856static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5857{
5858 int rc = VINF_SUCCESS;
5859 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5860 {
5861 uint32_t u32Val = 0;
5862 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5863 pMixedCtx->SysEnter.cs = u32Val;
5864 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5865 }
5866
5867 uint64_t u64Val = 0;
5868 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5869 {
5870 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5871 pMixedCtx->SysEnter.eip = u64Val;
5872 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5873 }
5874 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5875 {
5876 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5877 pMixedCtx->SysEnter.esp = u64Val;
5878 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5879 }
5880 return rc;
5881}
5882
5883
5884/**
5885 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5886 * the CPU back into the guest-CPU context.
5887 *
5888 * @returns VBox status code.
5889 * @param pVCpu Pointer to the VMCPU.
5890 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5891 * out-of-sync. Make sure to update the required fields
5892 * before using them.
5893 *
5894 * @remarks No-long-jump zone!!!
5895 */
5896static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5897{
5898#if HC_ARCH_BITS == 64
5899 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5900 {
5901 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5902 VMMRZCallRing3Disable(pVCpu);
5903 HM_DISABLE_PREEMPT_IF_NEEDED();
5904
5905 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5906 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5907 {
5908 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5909 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5910 }
5911
5912 HM_RESTORE_PREEMPT_IF_NEEDED();
5913 VMMRZCallRing3Enable(pVCpu);
5914 }
5915 else
5916 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5917#else
5918 NOREF(pMixedCtx);
5919 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5920#endif
5921
5922 return VINF_SUCCESS;
5923}
5924
5925
5926/**
5927 * Saves the auto load/store'd guest MSRs from the current VMCS into
5928 * the guest-CPU context.
5929 *
5930 * @returns VBox status code.
5931 * @param pVCpu Pointer to the VMCPU.
5932 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5933 * out-of-sync. Make sure to update the required fields
5934 * before using them.
5935 *
5936 * @remarks No-long-jump zone!!!
5937 */
5938static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5939{
5940 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5941 return VINF_SUCCESS;
5942
5943 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5944 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5945 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5946 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5947 {
5948 switch (pMsr->u32Msr)
5949 {
5950 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5951 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5952 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5953 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5954 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5955 default:
5956 {
5957 AssertFailed();
5958 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5959 }
5960 }
5961 }
5962
5963 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5964 return VINF_SUCCESS;
5965}
5966
5967
5968/**
5969 * Saves the guest control registers from the current VMCS into the guest-CPU
5970 * context.
5971 *
5972 * @returns VBox status code.
5973 * @param pVCpu Pointer to the VMCPU.
5974 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5975 * out-of-sync. Make sure to update the required fields
5976 * before using them.
5977 *
5978 * @remarks No-long-jump zone!!!
5979 */
5980static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5981{
5982 /* Guest CR0. Guest FPU. */
5983 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5984 AssertRCReturn(rc, rc);
5985
5986 /* Guest CR4. */
5987 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5988 AssertRCReturn(rc, rc);
5989
5990 /* Guest CR2 - updated always during the world-switch or in #PF. */
5991 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5992 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5993 {
5994 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5995 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5996
5997 PVM pVM = pVCpu->CTX_SUFF(pVM);
5998 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5999 || ( pVM->hm.s.fNestedPaging
6000 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6001 {
6002 uint64_t u64Val = 0;
6003 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6004 if (pMixedCtx->cr3 != u64Val)
6005 {
6006 CPUMSetGuestCR3(pVCpu, u64Val);
6007 if (VMMRZCallRing3IsEnabled(pVCpu))
6008 {
6009 PGMUpdateCR3(pVCpu, u64Val);
6010 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6011 }
6012 else
6013 {
6014 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6015 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6016 }
6017 }
6018
6019 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6020 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6021 {
6022 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6023 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6024 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6025 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6026
6027 if (VMMRZCallRing3IsEnabled(pVCpu))
6028 {
6029 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6030 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6031 }
6032 else
6033 {
6034 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6035 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6036 }
6037 }
6038 }
6039
6040 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6041 }
6042
6043 /*
6044 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6045 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6046 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6047 *
6048 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6049 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6050 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6051 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6052 *
6053 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6054 */
6055 if (VMMRZCallRing3IsEnabled(pVCpu))
6056 {
6057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6058 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6059
6060 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6061 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6062
6063 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6064 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6065 }
6066
6067 return rc;
6068}
6069
6070
6071/**
6072 * Reads a guest segment register from the current VMCS into the guest-CPU
6073 * context.
6074 *
6075 * @returns VBox status code.
6076 * @param pVCpu Pointer to the VMCPU.
6077 * @param idxSel Index of the selector in the VMCS.
6078 * @param idxLimit Index of the segment limit in the VMCS.
6079 * @param idxBase Index of the segment base in the VMCS.
6080 * @param idxAccess Index of the access rights of the segment in the VMCS.
6081 * @param pSelReg Pointer to the segment selector.
6082 *
6083 * @remarks No-long-jump zone!!!
6084 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6085 * macro as that takes care of whether to read from the VMCS cache or
6086 * not.
6087 */
6088DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6089 PCPUMSELREG pSelReg)
6090{
6091 NOREF(pVCpu);
6092
6093 uint32_t u32Val = 0;
6094 int rc = VMXReadVmcs32(idxSel, &u32Val);
6095 AssertRCReturn(rc, rc);
6096 pSelReg->Sel = (uint16_t)u32Val;
6097 pSelReg->ValidSel = (uint16_t)u32Val;
6098 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6099
6100 rc = VMXReadVmcs32(idxLimit, &u32Val);
6101 AssertRCReturn(rc, rc);
6102 pSelReg->u32Limit = u32Val;
6103
6104 uint64_t u64Val = 0;
6105 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6106 AssertRCReturn(rc, rc);
6107 pSelReg->u64Base = u64Val;
6108
6109 rc = VMXReadVmcs32(idxAccess, &u32Val);
6110 AssertRCReturn(rc, rc);
6111 pSelReg->Attr.u = u32Val;
6112
6113 /*
6114 * If VT-x marks the segment as unusable, most other bits remain undefined:
6115 * - For CS the L, D and G bits have meaning.
6116 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6117 * - For the remaining data segments no bits are defined.
6118 *
6119 * The present bit and the unusable bit has been observed to be set at the
6120 * same time (the selector was supposed to invalid as we started executing
6121 * a V8086 interrupt in ring-0).
6122 *
6123 * What should be important for the rest of the VBox code, is that the P bit is
6124 * cleared. Some of the other VBox code recognizes the unusable bit, but
6125 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6126 * safe side here, we'll strip off P and other bits we don't care about. If
6127 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6128 *
6129 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6130 */
6131 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6132 {
6133 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6134
6135 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6136 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6137 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6138
6139 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6140#ifdef DEBUG_bird
6141 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6142 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6143 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6144#endif
6145 }
6146 return VINF_SUCCESS;
6147}
6148
6149
6150#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6151# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6152 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6153 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6154#else
6155# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6156 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6157 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6158#endif
6159
6160
6161/**
6162 * Saves the guest segment registers from the current VMCS into the guest-CPU
6163 * context.
6164 *
6165 * @returns VBox status code.
6166 * @param pVCpu Pointer to the VMCPU.
6167 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6168 * out-of-sync. Make sure to update the required fields
6169 * before using them.
6170 *
6171 * @remarks No-long-jump zone!!!
6172 */
6173static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6174{
6175 /* Guest segment registers. */
6176 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6177 {
6178 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6179 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6180 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6181 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6182 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6183 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6184 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6185
6186 /* Restore segment attributes for real-on-v86 mode hack. */
6187 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6188 {
6189 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6190 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6191 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6192 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6193 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6194 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6195 }
6196 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6197 }
6198
6199 return VINF_SUCCESS;
6200}
6201
6202
6203/**
6204 * Saves the guest descriptor table registers and task register from the current
6205 * VMCS into the guest-CPU context.
6206 *
6207 * @returns VBox status code.
6208 * @param pVCpu Pointer to the VMCPU.
6209 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6210 * out-of-sync. Make sure to update the required fields
6211 * before using them.
6212 *
6213 * @remarks No-long-jump zone!!!
6214 */
6215static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6216{
6217 int rc = VINF_SUCCESS;
6218
6219 /* Guest LDTR. */
6220 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6221 {
6222 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6223 AssertRCReturn(rc, rc);
6224 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6225 }
6226
6227 /* Guest GDTR. */
6228 uint64_t u64Val = 0;
6229 uint32_t u32Val = 0;
6230 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6231 {
6232 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6233 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6234 pMixedCtx->gdtr.pGdt = u64Val;
6235 pMixedCtx->gdtr.cbGdt = u32Val;
6236 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6237 }
6238
6239 /* Guest IDTR. */
6240 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6241 {
6242 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6243 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6244 pMixedCtx->idtr.pIdt = u64Val;
6245 pMixedCtx->idtr.cbIdt = u32Val;
6246 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6247 }
6248
6249 /* Guest TR. */
6250 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6251 {
6252 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6253 AssertRCReturn(rc, rc);
6254
6255 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6256 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6257 {
6258 rc = VMXLOCAL_READ_SEG(TR, tr);
6259 AssertRCReturn(rc, rc);
6260 }
6261 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6262 }
6263 return rc;
6264}
6265
6266#undef VMXLOCAL_READ_SEG
6267
6268
6269/**
6270 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6271 * context.
6272 *
6273 * @returns VBox status code.
6274 * @param pVCpu Pointer to the VMCPU.
6275 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6276 * out-of-sync. Make sure to update the required fields
6277 * before using them.
6278 *
6279 * @remarks No-long-jump zone!!!
6280 */
6281static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282{
6283 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6284 {
6285 if (!pVCpu->hm.s.fUsingHyperDR7)
6286 {
6287 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6288 uint32_t u32Val;
6289 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6290 pMixedCtx->dr[7] = u32Val;
6291 }
6292
6293 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6294 }
6295 return VINF_SUCCESS;
6296}
6297
6298
6299/**
6300 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6301 *
6302 * @returns VBox status code.
6303 * @param pVCpu Pointer to the VMCPU.
6304 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6305 * out-of-sync. Make sure to update the required fields
6306 * before using them.
6307 *
6308 * @remarks No-long-jump zone!!!
6309 */
6310static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6311{
6312 NOREF(pMixedCtx);
6313
6314 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6315 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6316 return VINF_SUCCESS;
6317}
6318
6319
6320/**
6321 * Saves the entire guest state from the currently active VMCS into the
6322 * guest-CPU context. This essentially VMREADs all guest-data.
6323 *
6324 * @returns VBox status code.
6325 * @param pVCpu Pointer to the VMCPU.
6326 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6327 * out-of-sync. Make sure to update the required fields
6328 * before using them.
6329 */
6330static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6331{
6332 Assert(pVCpu);
6333 Assert(pMixedCtx);
6334
6335 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6336 return VINF_SUCCESS;
6337
6338 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6339 again on the ring-3 callback path, there is no real need to. */
6340 if (VMMRZCallRing3IsEnabled(pVCpu))
6341 VMMR0LogFlushDisable(pVCpu);
6342 else
6343 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6344 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6345
6346 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6347 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6348
6349 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6350 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6351
6352 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6353 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6354
6355 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6356 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6357
6358 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6359 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6360
6361 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6362 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6363
6364 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6365 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6366
6367 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6368 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6369
6370 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6371 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6372
6373 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6374 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6375
6376 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6377 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6378
6379 if (VMMRZCallRing3IsEnabled(pVCpu))
6380 VMMR0LogFlushEnable(pVCpu);
6381
6382 return rc;
6383}
6384
6385
6386/**
6387 * Check per-VM and per-VCPU force flag actions that require us to go back to
6388 * ring-3 for one reason or another.
6389 *
6390 * @returns VBox status code (information status code included).
6391 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6392 * ring-3.
6393 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6394 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6395 * interrupts)
6396 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6397 * all EMTs to be in ring-3.
6398 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6399 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6400 * to the EM loop.
6401 *
6402 * @param pVM Pointer to the VM.
6403 * @param pVCpu Pointer to the VMCPU.
6404 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6405 * out-of-sync. Make sure to update the required fields
6406 * before using them.
6407 */
6408static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6409{
6410 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6411
6412 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6413 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6414 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6415 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6416 {
6417 /* We need the control registers now, make sure the guest-CPU context is updated. */
6418 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6419 AssertRCReturn(rc3, rc3);
6420
6421 /* Pending HM CR3 sync. */
6422 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6423 {
6424 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6425 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6426 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6427 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6428 }
6429
6430 /* Pending HM PAE PDPEs. */
6431 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6432 {
6433 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6434 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6435 }
6436
6437 /* Pending PGM C3 sync. */
6438 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6439 {
6440 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6441 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6442 if (rc2 != VINF_SUCCESS)
6443 {
6444 AssertRC(rc2);
6445 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6446 return rc2;
6447 }
6448 }
6449
6450 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6451 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6452 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6453 {
6454 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6455 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6456 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6457 return rc2;
6458 }
6459
6460 /* Pending VM request packets, such as hardware interrupts. */
6461 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6462 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6463 {
6464 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6465 return VINF_EM_PENDING_REQUEST;
6466 }
6467
6468 /* Pending PGM pool flushes. */
6469 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6470 {
6471 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6472 return VINF_PGM_POOL_FLUSH_PENDING;
6473 }
6474
6475 /* Pending DMA requests. */
6476 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6477 {
6478 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6479 return VINF_EM_RAW_TO_R3;
6480 }
6481 }
6482
6483 return VINF_SUCCESS;
6484}
6485
6486
6487/**
6488 * Converts any TRPM trap into a pending HM event. This is typically used when
6489 * entering from ring-3 (not longjmp returns).
6490 *
6491 * @param pVCpu Pointer to the VMCPU.
6492 */
6493static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6494{
6495 Assert(TRPMHasTrap(pVCpu));
6496 Assert(!pVCpu->hm.s.Event.fPending);
6497
6498 uint8_t uVector;
6499 TRPMEVENT enmTrpmEvent;
6500 RTGCUINT uErrCode;
6501 RTGCUINTPTR GCPtrFaultAddress;
6502 uint8_t cbInstr;
6503
6504 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6505 AssertRC(rc);
6506
6507 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6508 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6509 if (enmTrpmEvent == TRPM_TRAP)
6510 {
6511 switch (uVector)
6512 {
6513 case X86_XCPT_BP:
6514 case X86_XCPT_OF:
6515 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6516 break;
6517
6518 case X86_XCPT_PF:
6519 case X86_XCPT_DF:
6520 case X86_XCPT_TS:
6521 case X86_XCPT_NP:
6522 case X86_XCPT_SS:
6523 case X86_XCPT_GP:
6524 case X86_XCPT_AC:
6525 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6526 /* no break! */
6527 default:
6528 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6529 break;
6530 }
6531 }
6532 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6533 {
6534 if (uVector == X86_XCPT_NMI)
6535 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6536 else
6537 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6538 }
6539 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6540 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6541 else
6542 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6543
6544 rc = TRPMResetTrap(pVCpu);
6545 AssertRC(rc);
6546 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6547 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6548
6549 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6550 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6551}
6552
6553
6554/**
6555 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6556 * VT-x to execute any instruction.
6557 *
6558 * @param pvCpu Pointer to the VMCPU.
6559 */
6560static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6561{
6562 Assert(pVCpu->hm.s.Event.fPending);
6563
6564 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6565 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6566 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6567 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6568
6569 /* If a trap was already pending, we did something wrong! */
6570 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6571
6572 TRPMEVENT enmTrapType;
6573 switch (uVectorType)
6574 {
6575 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6576 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6577 enmTrapType = TRPM_HARDWARE_INT;
6578 break;
6579
6580 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6581 enmTrapType = TRPM_SOFTWARE_INT;
6582 break;
6583
6584 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6585 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6586 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6587 enmTrapType = TRPM_TRAP;
6588 break;
6589
6590 default:
6591 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6592 enmTrapType = TRPM_32BIT_HACK;
6593 break;
6594 }
6595
6596 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6597
6598 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6599 AssertRC(rc);
6600
6601 if (fErrorCodeValid)
6602 TRPMSetErrorCode(pVCpu, uErrorCode);
6603
6604 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6605 && uVector == X86_XCPT_PF)
6606 {
6607 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6608 }
6609 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6610 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6611 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6612 {
6613 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6614 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6615 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6616 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6617 }
6618 pVCpu->hm.s.Event.fPending = false;
6619}
6620
6621
6622/**
6623 * Does the necessary state syncing before returning to ring-3 for any reason
6624 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6625 *
6626 * @returns VBox status code.
6627 * @param pVM Pointer to the VM.
6628 * @param pVCpu Pointer to the VMCPU.
6629 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6630 * be out-of-sync. Make sure to update the required
6631 * fields before using them.
6632 * @param fSaveGuestState Whether to save the guest state or not.
6633 *
6634 * @remarks No-long-jmp zone!!!
6635 */
6636static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6637{
6638 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6639 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6640
6641 RTCPUID idCpu = RTMpCpuId();
6642 Log4Func(("HostCpuId=%u\n", idCpu));
6643
6644 /*
6645 * !!! IMPORTANT !!!
6646 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6647 */
6648
6649 /* Save the guest state if necessary. */
6650 if ( fSaveGuestState
6651 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6652 {
6653 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6654 AssertRCReturn(rc, rc);
6655 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6656 }
6657
6658 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6659 if (CPUMIsGuestFPUStateActive(pVCpu))
6660 {
6661 /* We shouldn't reload CR0 without saving it first. */
6662 if (!fSaveGuestState)
6663 {
6664 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6665 AssertRCReturn(rc, rc);
6666 }
6667 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6668 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6669 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6670 }
6671
6672 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6673#ifdef VBOX_STRICT
6674 if (CPUMIsHyperDebugStateActive(pVCpu))
6675 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6676#endif
6677 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6678 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6679 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6680 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6681
6682#if HC_ARCH_BITS == 64
6683 /* Restore host-state bits that VT-x only restores partially. */
6684 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6685 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6686 {
6687 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6688 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6689 }
6690 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6691#endif
6692
6693#if HC_ARCH_BITS == 64
6694 /* Restore the host MSRs as we're leaving VT-x context. */
6695 if ( pVM->hm.s.fAllow64BitGuests
6696 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6697 {
6698 /* We shouldn't reload the guest MSRs without saving it first. */
6699 if (!fSaveGuestState)
6700 {
6701 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6702 AssertRCReturn(rc, rc);
6703 }
6704 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6705 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6706 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6707 }
6708#endif
6709
6710 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6711 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6712
6713 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6714 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6715 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6716 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6717 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6718 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6719 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6720 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6721
6722 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6723
6724 /** @todo This partially defeats the purpose of having preemption hooks.
6725 * The problem is, deregistering the hooks should be moved to a place that
6726 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6727 * context.
6728 */
6729 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6730 {
6731 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6732 AssertRCReturn(rc, rc);
6733
6734 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6735 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6736 }
6737 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6738 NOREF(idCpu);
6739
6740 return VINF_SUCCESS;
6741}
6742
6743
6744/**
6745 * Leaves the VT-x session.
6746 *
6747 * @returns VBox status code.
6748 * @param pVM Pointer to the VM.
6749 * @param pVCpu Pointer to the VMCPU.
6750 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6751 * out-of-sync. Make sure to update the required fields
6752 * before using them.
6753 *
6754 * @remarks No-long-jmp zone!!!
6755 */
6756DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6757{
6758 HM_DISABLE_PREEMPT_IF_NEEDED();
6759 HMVMX_ASSERT_CPU_SAFE();
6760 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6761 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6762
6763 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6764 and done this from the VMXR0ThreadCtxCallback(). */
6765 if (!pVCpu->hm.s.fLeaveDone)
6766 {
6767 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6768 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6769 pVCpu->hm.s.fLeaveDone = true;
6770 }
6771
6772 /*
6773 * !!! IMPORTANT !!!
6774 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6775 */
6776
6777 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6778 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6779 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6780 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6781 VMMR0ThreadCtxHooksDeregister(pVCpu);
6782
6783 /* Leave HM context. This takes care of local init (term). */
6784 int rc = HMR0LeaveCpu(pVCpu);
6785
6786 HM_RESTORE_PREEMPT_IF_NEEDED();
6787
6788 return rc;
6789}
6790
6791
6792/**
6793 * Does the necessary state syncing before doing a longjmp to ring-3.
6794 *
6795 * @returns VBox status code.
6796 * @param pVM Pointer to the VM.
6797 * @param pVCpu Pointer to the VMCPU.
6798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6799 * out-of-sync. Make sure to update the required fields
6800 * before using them.
6801 *
6802 * @remarks No-long-jmp zone!!!
6803 */
6804DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6805{
6806 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6807}
6808
6809
6810/**
6811 * Take necessary actions before going back to ring-3.
6812 *
6813 * An action requires us to go back to ring-3. This function does the necessary
6814 * steps before we can safely return to ring-3. This is not the same as longjmps
6815 * to ring-3, this is voluntary and prepares the guest so it may continue
6816 * executing outside HM (recompiler/IEM).
6817 *
6818 * @returns VBox status code.
6819 * @param pVM Pointer to the VM.
6820 * @param pVCpu Pointer to the VMCPU.
6821 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6822 * out-of-sync. Make sure to update the required fields
6823 * before using them.
6824 * @param rcExit The reason for exiting to ring-3. Can be
6825 * VINF_VMM_UNKNOWN_RING3_CALL.
6826 */
6827static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6828{
6829 Assert(pVM);
6830 Assert(pVCpu);
6831 Assert(pMixedCtx);
6832 HMVMX_ASSERT_PREEMPT_SAFE();
6833
6834 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6835 {
6836 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6837 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6838 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6839 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6840 }
6841
6842 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6843 VMMRZCallRing3Disable(pVCpu);
6844 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6845
6846 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6847 if (pVCpu->hm.s.Event.fPending)
6848 {
6849 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6850 Assert(!pVCpu->hm.s.Event.fPending);
6851 }
6852
6853 /* Save guest state and restore host state bits. */
6854 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6855 AssertRCReturn(rc, rc);
6856 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6857
6858 /* Sync recompiler state. */
6859 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6860 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6861 | CPUM_CHANGED_LDTR
6862 | CPUM_CHANGED_GDTR
6863 | CPUM_CHANGED_IDTR
6864 | CPUM_CHANGED_TR
6865 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6866 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6867 if ( pVM->hm.s.fNestedPaging
6868 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6869 {
6870 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6871 }
6872
6873 Assert(!pVCpu->hm.s.fClearTrapFlag);
6874
6875 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6876 if (rcExit != VINF_EM_RAW_INTERRUPT)
6877 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6878
6879 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6880
6881 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6882 VMMRZCallRing3RemoveNotification(pVCpu);
6883 VMMRZCallRing3Enable(pVCpu);
6884
6885 return rc;
6886}
6887
6888
6889/**
6890 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6891 * longjump to ring-3 and possibly get preempted.
6892 *
6893 * @returns VBox status code.
6894 * @param pVCpu Pointer to the VMCPU.
6895 * @param enmOperation The operation causing the ring-3 longjump.
6896 * @param pvUser Opaque pointer to the guest-CPU context. The data
6897 * may be out-of-sync. Make sure to update the required
6898 * fields before using them.
6899 */
6900DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6901{
6902 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6903 {
6904 /*
6905 * !!! IMPORTANT !!!
6906 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6907 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6908 */
6909 VMMRZCallRing3RemoveNotification(pVCpu);
6910 VMMRZCallRing3Disable(pVCpu);
6911 HM_DISABLE_PREEMPT_IF_NEEDED();
6912
6913 PVM pVM = pVCpu->CTX_SUFF(pVM);
6914 if (CPUMIsGuestFPUStateActive(pVCpu))
6915 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6916
6917 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6918
6919#if HC_ARCH_BITS == 64
6920 /* Restore host-state bits that VT-x only restores partially. */
6921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6923 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6924 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6925
6926 /* Restore the host MSRs as we're leaving VT-x context. */
6927 if ( pVM->hm.s.fAllow64BitGuests
6928 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6929 {
6930 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6931 }
6932#endif
6933 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6934 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6935 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6936 {
6937 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6938 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6939 }
6940
6941 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6942 VMMR0ThreadCtxHooksDeregister(pVCpu);
6943
6944 HMR0LeaveCpu(pVCpu);
6945 HM_RESTORE_PREEMPT_IF_NEEDED();
6946 return VINF_SUCCESS;
6947 }
6948
6949 Assert(pVCpu);
6950 Assert(pvUser);
6951 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6952 HMVMX_ASSERT_PREEMPT_SAFE();
6953
6954 VMMRZCallRing3Disable(pVCpu);
6955 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6956
6957 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6958 enmOperation));
6959
6960 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6961 AssertRCReturn(rc, rc);
6962
6963 VMMRZCallRing3Enable(pVCpu);
6964 return VINF_SUCCESS;
6965}
6966
6967
6968/**
6969 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6970 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6971 *
6972 * @param pVCpu Pointer to the VMCPU.
6973 */
6974DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6975{
6976 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6977 {
6978 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6979 {
6980 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6981 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6982 AssertRC(rc);
6983 Log4(("Setup interrupt-window exiting\n"));
6984 }
6985 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6986}
6987
6988
6989/**
6990 * Clears the interrupt-window exiting control in the VMCS.
6991 *
6992 * @param pVCpu Pointer to the VMCPU.
6993 */
6994DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
6995{
6996 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
6997 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6998 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6999 AssertRC(rc);
7000 Log4(("Cleared interrupt-window exiting\n"));
7001}
7002
7003
7004/**
7005 * Evaluates the event to be delivered to the guest and sets it as the pending
7006 * event.
7007 *
7008 * @param pVCpu Pointer to the VMCPU.
7009 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7010 * out-of-sync. Make sure to update the required fields
7011 * before using them.
7012 */
7013static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7014{
7015 Assert(!pVCpu->hm.s.Event.fPending);
7016
7017 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7018 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7019 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7020 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7021
7022 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7023 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7024 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7025 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7026 Assert(!TRPMHasTrap(pVCpu));
7027
7028 /** @todo SMI. SMIs take priority over NMIs. */
7029 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7030 {
7031 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7032 if ( !fBlockMovSS
7033 && !fBlockSti)
7034 {
7035 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7036 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7037 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7038 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7039
7040 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7041 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7042 }
7043 else
7044 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7045 }
7046 /*
7047 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7048 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7049 */
7050 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7051 && !pVCpu->hm.s.fSingleInstruction)
7052 {
7053 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7054 AssertRC(rc);
7055 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7056 if ( !fBlockInt
7057 && !fBlockSti
7058 && !fBlockMovSS)
7059 {
7060 uint8_t u8Interrupt;
7061 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7062 if (RT_SUCCESS(rc))
7063 {
7064 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7065 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7066 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7067
7068 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7069 }
7070 else
7071 {
7072 /** @todo Does this actually happen? If not turn it into an assertion. */
7073 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7074 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7075 }
7076 }
7077 else
7078 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7079 }
7080}
7081
7082
7083/**
7084 * Sets a pending-debug exception to be delivered to the guest if the guest is
7085 * single-stepping.
7086 *
7087 * @param pVCpu Pointer to the VMCPU.
7088 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7089 * out-of-sync. Make sure to update the required fields
7090 * before using them.
7091 */
7092DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7093{
7094 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7095 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7096 {
7097 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7098 AssertRC(rc);
7099 }
7100}
7101
7102
7103/**
7104 * Injects any pending events into the guest if the guest is in a state to
7105 * receive them.
7106 *
7107 * @returns VBox status code (informational status codes included).
7108 * @param pVCpu Pointer to the VMCPU.
7109 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7110 * out-of-sync. Make sure to update the required fields
7111 * before using them.
7112 */
7113static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7114{
7115 HMVMX_ASSERT_PREEMPT_SAFE();
7116 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7117
7118 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7119 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7120 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7121 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7122
7123 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7124 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7125 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7126 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7127 Assert(!TRPMHasTrap(pVCpu));
7128
7129 int rc = VINF_SUCCESS;
7130 if (pVCpu->hm.s.Event.fPending)
7131 {
7132 /*
7133 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7134 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7135 * ended up enabling interrupts outside VT-x.
7136 */
7137 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7138 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7139 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7140 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7141 {
7142 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7143 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7144 }
7145#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
7146 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7147 {
7148 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7149 AssertRCReturn(rc, rc);
7150 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7151 Assert(!fBlockInt);
7152 Assert(!fBlockSti);
7153 Assert(!fBlockMovSS);
7154 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7155 }
7156 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7157 {
7158 Assert(!fBlockSti);
7159 Assert(!fBlockMovSS);
7160 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7161 }
7162#endif
7163 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7164 (uint8_t)uIntType));
7165 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7166 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7167 AssertRCReturn(rc, rc);
7168
7169 /* Update the interruptibility-state as it could have been changed by
7170 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7171 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7172 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7173
7174#ifdef VBOX_WITH_STATISTICS
7175 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7176 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7177 else
7178 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7179#endif
7180 }
7181
7182 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7183 if ( fBlockSti
7184 || fBlockMovSS)
7185 {
7186 if ( !pVCpu->hm.s.fSingleInstruction
7187 && !DBGFIsStepping(pVCpu))
7188 {
7189 /*
7190 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7191 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7192 * See Intel spec. 27.3.4 "Saving Non-Register State".
7193 */
7194 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7195 AssertRCReturn(rc2, rc2);
7196 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7197 }
7198 else if (pMixedCtx->eflags.Bits.u1TF)
7199 {
7200 /*
7201 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7202 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7203 */
7204 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7205 uIntrState = 0;
7206 }
7207 }
7208
7209 /*
7210 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7211 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7212 */
7213 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7214 AssertRC(rc2);
7215
7216 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7217 NOREF(fBlockMovSS); NOREF(fBlockSti);
7218 return rc;
7219}
7220
7221
7222/**
7223 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7224 *
7225 * @param pVCpu Pointer to the VMCPU.
7226 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7227 * out-of-sync. Make sure to update the required fields
7228 * before using them.
7229 */
7230DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7231{
7232 NOREF(pMixedCtx);
7233 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7234 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7235}
7236
7237
7238/**
7239 * Injects a double-fault (#DF) exception into the VM.
7240 *
7241 * @returns VBox status code (informational status code included).
7242 * @param pVCpu Pointer to the VMCPU.
7243 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7244 * out-of-sync. Make sure to update the required fields
7245 * before using them.
7246 */
7247DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7248{
7249 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7250 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7251 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7252 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7253 puIntrState);
7254}
7255
7256
7257/**
7258 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7259 *
7260 * @param pVCpu Pointer to the VMCPU.
7261 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7262 * out-of-sync. Make sure to update the required fields
7263 * before using them.
7264 */
7265DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7266{
7267 NOREF(pMixedCtx);
7268 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7269 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7270 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7271}
7272
7273
7274/**
7275 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7276 *
7277 * @param pVCpu Pointer to the VMCPU.
7278 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7279 * out-of-sync. Make sure to update the required fields
7280 * before using them.
7281 * @param cbInstr The value of RIP that is to be pushed on the guest
7282 * stack.
7283 */
7284DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7285{
7286 NOREF(pMixedCtx);
7287 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7288 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7289 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7290}
7291
7292
7293/**
7294 * Injects a general-protection (#GP) fault into the VM.
7295 *
7296 * @returns VBox status code (informational status code included).
7297 * @param pVCpu Pointer to the VMCPU.
7298 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7299 * out-of-sync. Make sure to update the required fields
7300 * before using them.
7301 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7302 * mode, i.e. in real-mode it's not valid).
7303 * @param u32ErrorCode The error code associated with the #GP.
7304 */
7305DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7306 uint32_t *puIntrState)
7307{
7308 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7309 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7310 if (fErrorCodeValid)
7311 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7312 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7313 puIntrState);
7314}
7315
7316
7317/**
7318 * Sets a general-protection (#GP) exception as pending-for-injection into the
7319 * VM.
7320 *
7321 * @param pVCpu Pointer to the VMCPU.
7322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7323 * out-of-sync. Make sure to update the required fields
7324 * before using them.
7325 * @param u32ErrorCode The error code associated with the #GP.
7326 */
7327DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7328{
7329 NOREF(pMixedCtx);
7330 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7331 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7332 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7333 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7334}
7335
7336
7337/**
7338 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7339 *
7340 * @param pVCpu Pointer to the VMCPU.
7341 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7342 * out-of-sync. Make sure to update the required fields
7343 * before using them.
7344 * @param uVector The software interrupt vector number.
7345 * @param cbInstr The value of RIP that is to be pushed on the guest
7346 * stack.
7347 */
7348DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7349{
7350 NOREF(pMixedCtx);
7351 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7352 if ( uVector == X86_XCPT_BP
7353 || uVector == X86_XCPT_OF)
7354 {
7355 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7356 }
7357 else
7358 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7359 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7360}
7361
7362
7363/**
7364 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7365 * stack.
7366 *
7367 * @returns VBox status code (information status code included).
7368 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7369 * @param pVM Pointer to the VM.
7370 * @param pMixedCtx Pointer to the guest-CPU context.
7371 * @param uValue The value to push to the guest stack.
7372 */
7373DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7374{
7375 /*
7376 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7377 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7378 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7379 */
7380 if (pMixedCtx->sp == 1)
7381 return VINF_EM_RESET;
7382 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7383 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7384 AssertRCReturn(rc, rc);
7385 return rc;
7386}
7387
7388
7389/**
7390 * Injects an event into the guest upon VM-entry by updating the relevant fields
7391 * in the VM-entry area in the VMCS.
7392 *
7393 * @returns VBox status code (informational error codes included).
7394 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7395 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7396 *
7397 * @param pVCpu Pointer to the VMCPU.
7398 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7399 * be out-of-sync. Make sure to update the required
7400 * fields before using them.
7401 * @param u64IntInfo The VM-entry interruption-information field.
7402 * @param cbInstr The VM-entry instruction length in bytes (for
7403 * software interrupts, exceptions and privileged
7404 * software exceptions).
7405 * @param u32ErrCode The VM-entry exception error code.
7406 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7407 * @param puIntrState Pointer to the current guest interruptibility-state.
7408 * This interruptibility-state will be updated if
7409 * necessary. This cannot not be NULL.
7410 *
7411 * @remarks Requires CR0!
7412 * @remarks No-long-jump zone!!!
7413 */
7414static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7415 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7416{
7417 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7418 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7419 Assert(puIntrState);
7420 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7421
7422 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7423 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7424
7425#ifdef VBOX_STRICT
7426 /* Validate the error-code-valid bit for hardware exceptions. */
7427 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7428 {
7429 switch (uVector)
7430 {
7431 case X86_XCPT_PF:
7432 case X86_XCPT_DF:
7433 case X86_XCPT_TS:
7434 case X86_XCPT_NP:
7435 case X86_XCPT_SS:
7436 case X86_XCPT_GP:
7437 case X86_XCPT_AC:
7438 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7439 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7440 /* fallthru */
7441 default:
7442 break;
7443 }
7444 }
7445#endif
7446
7447 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7448 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7449 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7450
7451 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7452
7453 /* We require CR0 to check if the guest is in real-mode. */
7454 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7455 AssertRCReturn(rc, rc);
7456
7457 /*
7458 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7459 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7460 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7461 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7462 */
7463 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7464 {
7465 PVM pVM = pVCpu->CTX_SUFF(pVM);
7466 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7467 {
7468 Assert(PDMVmmDevHeapIsEnabled(pVM));
7469 Assert(pVM->hm.s.vmx.pRealModeTSS);
7470
7471 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7472 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7473 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7474 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7475 AssertRCReturn(rc, rc);
7476 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7477
7478 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7479 const size_t cbIdtEntry = sizeof(X86IDTR16);
7480 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7481 {
7482 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7483 if (uVector == X86_XCPT_DF)
7484 return VINF_EM_RESET;
7485 else if (uVector == X86_XCPT_GP)
7486 {
7487 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7488 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7489 }
7490
7491 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7492 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7493 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7494 }
7495
7496 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7497 uint16_t uGuestIp = pMixedCtx->ip;
7498 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7499 {
7500 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7501 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7502 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7503 }
7504 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7505 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7506
7507 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7508 X86IDTR16 IdtEntry;
7509 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7510 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7511 AssertRCReturn(rc, rc);
7512
7513 /* Construct the stack frame for the interrupt/exception handler. */
7514 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7515 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7516 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7517 AssertRCReturn(rc, rc);
7518
7519 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7520 if (rc == VINF_SUCCESS)
7521 {
7522 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7523 pMixedCtx->rip = IdtEntry.offSel;
7524 pMixedCtx->cs.Sel = IdtEntry.uSel;
7525 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7526 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7527 && uVector == X86_XCPT_PF)
7528 {
7529 pMixedCtx->cr2 = GCPtrFaultAddress;
7530 }
7531
7532 /* If any other guest-state bits are changed here, make sure to update
7533 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7534 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7535 | HM_CHANGED_GUEST_RIP
7536 | HM_CHANGED_GUEST_RFLAGS
7537 | HM_CHANGED_GUEST_RSP);
7538
7539 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7540 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7541 {
7542 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7543 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7544 Log4(("Clearing inhibition due to STI.\n"));
7545 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7546 }
7547 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7548
7549 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7550 it, if we are returning to ring-3 before executing guest code. */
7551 pVCpu->hm.s.Event.fPending = false;
7552 }
7553 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7554 return rc;
7555 }
7556 else
7557 {
7558 /*
7559 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7560 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7561 */
7562 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7563 }
7564 }
7565
7566 /* Validate. */
7567 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7568 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7569 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7570
7571 /* Inject. */
7572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7573 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7574 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7575 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7576
7577 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7578 && uVector == X86_XCPT_PF)
7579 {
7580 pMixedCtx->cr2 = GCPtrFaultAddress;
7581 }
7582
7583 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7584 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7585
7586 AssertRCReturn(rc, rc);
7587 return rc;
7588}
7589
7590
7591/**
7592 * Clears the interrupt-window exiting control in the VMCS and if necessary
7593 * clears the current event in the VMCS as well.
7594 *
7595 * @returns VBox status code.
7596 * @param pVCpu Pointer to the VMCPU.
7597 *
7598 * @remarks Use this function only to clear events that have not yet been
7599 * delivered to the guest but are injected in the VMCS!
7600 * @remarks No-long-jump zone!!!
7601 */
7602static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7603{
7604 int rc;
7605 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7606
7607 /* Clear interrupt-window exiting control. */
7608 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7609 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7610
7611 if (!pVCpu->hm.s.Event.fPending)
7612 return;
7613
7614#ifdef VBOX_STRICT
7615 uint32_t u32EntryInfo;
7616 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7617 AssertRC(rc);
7618 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7619#endif
7620
7621 /* Clear the entry-interruption field (including the valid bit). */
7622 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7623 AssertRC(rc);
7624
7625 /* Clear the pending debug exception field. */
7626 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7627 AssertRC(rc);
7628
7629 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
7630 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
7631}
7632
7633
7634/**
7635 * Enters the VT-x session.
7636 *
7637 * @returns VBox status code.
7638 * @param pVM Pointer to the VM.
7639 * @param pVCpu Pointer to the VMCPU.
7640 * @param pCpu Pointer to the CPU info struct.
7641 */
7642VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7643{
7644 AssertPtr(pVM);
7645 AssertPtr(pVCpu);
7646 Assert(pVM->hm.s.vmx.fSupported);
7647 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7648 NOREF(pCpu); NOREF(pVM);
7649
7650 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7651 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7652
7653#ifdef VBOX_STRICT
7654 /* Make sure we're in VMX root mode. */
7655 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7656 if (!(u32HostCR4 & X86_CR4_VMXE))
7657 {
7658 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7659 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7660 }
7661#endif
7662
7663 /*
7664 * Load the VCPU's VMCS as the current (and active) one.
7665 */
7666 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7667 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7668 if (RT_FAILURE(rc))
7669 return rc;
7670
7671 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7672 pVCpu->hm.s.fLeaveDone = false;
7673 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7674
7675 return VINF_SUCCESS;
7676}
7677
7678
7679/**
7680 * The thread-context callback (only on platforms which support it).
7681 *
7682 * @param enmEvent The thread-context event.
7683 * @param pVCpu Pointer to the VMCPU.
7684 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7685 * @thread EMT(pVCpu)
7686 */
7687VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7688{
7689 NOREF(fGlobalInit);
7690
7691 switch (enmEvent)
7692 {
7693 case RTTHREADCTXEVENT_PREEMPTING:
7694 {
7695 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7696 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7697 VMCPU_ASSERT_EMT(pVCpu);
7698
7699 PVM pVM = pVCpu->CTX_SUFF(pVM);
7700 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7701
7702 /* No longjmps (logger flushes, locks) in this fragile context. */
7703 VMMRZCallRing3Disable(pVCpu);
7704 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7705
7706 /*
7707 * Restore host-state (FPU, debug etc.)
7708 */
7709 if (!pVCpu->hm.s.fLeaveDone)
7710 {
7711 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7712 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7713 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7714 pVCpu->hm.s.fLeaveDone = true;
7715 }
7716
7717 /* Leave HM context, takes care of local init (term). */
7718 int rc = HMR0LeaveCpu(pVCpu);
7719 AssertRC(rc); NOREF(rc);
7720
7721 /* Restore longjmp state. */
7722 VMMRZCallRing3Enable(pVCpu);
7723 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7724 break;
7725 }
7726
7727 case RTTHREADCTXEVENT_RESUMED:
7728 {
7729 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7730 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7731 VMCPU_ASSERT_EMT(pVCpu);
7732
7733 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7734 VMMRZCallRing3Disable(pVCpu);
7735 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7736
7737 /* Initialize the bare minimum state required for HM. This takes care of
7738 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7739 int rc = HMR0EnterCpu(pVCpu);
7740 AssertRC(rc);
7741 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7742
7743 /* Load the active VMCS as the current one. */
7744 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7745 {
7746 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7747 AssertRC(rc); NOREF(rc);
7748 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7749 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7750 }
7751 pVCpu->hm.s.fLeaveDone = false;
7752
7753 /* Restore longjmp state. */
7754 VMMRZCallRing3Enable(pVCpu);
7755 break;
7756 }
7757
7758 default:
7759 break;
7760 }
7761}
7762
7763
7764/**
7765 * Saves the host state in the VMCS host-state.
7766 * Sets up the VM-exit MSR-load area.
7767 *
7768 * The CPU state will be loaded from these fields on every successful VM-exit.
7769 *
7770 * @returns VBox status code.
7771 * @param pVM Pointer to the VM.
7772 * @param pVCpu Pointer to the VMCPU.
7773 *
7774 * @remarks No-long-jump zone!!!
7775 */
7776static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7777{
7778 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7779
7780 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7781 return VINF_SUCCESS;
7782
7783 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7784 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7785
7786 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7787 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7788
7789 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7790 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7791
7792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7793 return rc;
7794}
7795
7796
7797/**
7798 * Saves the host state in the VMCS host-state.
7799 *
7800 * @returns VBox status code.
7801 * @param pVM Pointer to the VM.
7802 * @param pVCpu Pointer to the VMCPU.
7803 *
7804 * @remarks No-long-jump zone!!!
7805 */
7806VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7807{
7808 AssertPtr(pVM);
7809 AssertPtr(pVCpu);
7810
7811 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7812
7813 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7814 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7815 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7816 return hmR0VmxSaveHostState(pVM, pVCpu);
7817}
7818
7819
7820/**
7821 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7822 * loaded from these fields on every successful VM-entry.
7823 *
7824 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7825 * Sets up the VM-entry controls.
7826 * Sets up the appropriate VMX non-root function to execute guest code based on
7827 * the guest CPU mode.
7828 *
7829 * @returns VBox status code.
7830 * @param pVM Pointer to the VM.
7831 * @param pVCpu Pointer to the VMCPU.
7832 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7833 * out-of-sync. Make sure to update the required fields
7834 * before using them.
7835 *
7836 * @remarks No-long-jump zone!!!
7837 */
7838static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7839{
7840 AssertPtr(pVM);
7841 AssertPtr(pVCpu);
7842 AssertPtr(pMixedCtx);
7843 HMVMX_ASSERT_PREEMPT_SAFE();
7844
7845#ifdef LOG_ENABLED
7846 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7847 * probably not initialized yet? Anyway this will do for now.
7848 *
7849 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7850 * interface and disable ring-3 calls when thread-context hooks are not
7851 * available. */
7852 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7853 VMMR0LogFlushDisable(pVCpu);
7854#endif
7855
7856 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7857
7858 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7859
7860 /* Determine real-on-v86 mode. */
7861 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7862 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7863 && CPUMIsGuestInRealModeEx(pMixedCtx))
7864 {
7865 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7866 }
7867
7868 /*
7869 * Load the guest-state into the VMCS.
7870 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7871 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7872 */
7873 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7874 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7875
7876 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7877 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7878 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7879
7880 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7881 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7882 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7883
7884 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7885 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7886
7887 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7888 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7889
7890 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7891 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7892 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7893
7894 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7895 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7896
7897 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7898 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7899
7900 /*
7901 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7902 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7903 */
7904 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7905 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7906
7907 /* Clear any unused and reserved bits. */
7908 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7909
7910#ifdef LOG_ENABLED
7911 /* Only reenable log-flushing if the caller has it enabled. */
7912 if (!fCallerDisabledLogFlush)
7913 VMMR0LogFlushEnable(pVCpu);
7914#endif
7915
7916 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7917 return rc;
7918}
7919
7920
7921/**
7922 * Loads the state shared between the host and guest into the VMCS.
7923 *
7924 * @param pVM Pointer to the VM.
7925 * @param pVCpu Pointer to the VMCPU.
7926 * @param pCtx Pointer to the guest-CPU context.
7927 *
7928 * @remarks No-long-jump zone!!!
7929 */
7930static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7931{
7932 NOREF(pVM);
7933
7934 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7935 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7936
7937 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7938 {
7939 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7940 AssertRC(rc);
7941 }
7942
7943 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7944 {
7945 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7946 AssertRC(rc);
7947
7948 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7949 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7950 {
7951 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7952 AssertRC(rc);
7953 }
7954 }
7955
7956 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7957 {
7958#if HC_ARCH_BITS == 64
7959 if (pVM->hm.s.fAllow64BitGuests)
7960 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7961#endif
7962 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7963 }
7964
7965 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7966 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7967}
7968
7969
7970/**
7971 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7972 *
7973 * @param pVM Pointer to the VM.
7974 * @param pVCpu Pointer to the VMCPU.
7975 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7976 * out-of-sync. Make sure to update the required fields
7977 * before using them.
7978 */
7979DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7980{
7981 HMVMX_ASSERT_PREEMPT_SAFE();
7982
7983 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7984#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7985 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7986#endif
7987
7988 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7989 {
7990 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7991 AssertRC(rc);
7992 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7993 }
7994 else if (HMCPU_CF_VALUE(pVCpu))
7995 {
7996 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7997 AssertRC(rc);
7998 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7999 }
8000
8001 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8002 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8003 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8004 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8005
8006#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8007 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8008 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8009 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8010#endif
8011}
8012
8013
8014/**
8015 * Does the preparations before executing guest code in VT-x.
8016 *
8017 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8018 * recompiler. We must be cautious what we do here regarding committing
8019 * guest-state information into the VMCS assuming we assuredly execute the
8020 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8021 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8022 * so that the recompiler can (and should) use them when it resumes guest
8023 * execution. Otherwise such operations must be done when we can no longer
8024 * exit to ring-3.
8025 *
8026 * @returns Strict VBox status code.
8027 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8028 * have been disabled.
8029 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8030 * double-fault into the guest.
8031 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8032 *
8033 * @param pVM Pointer to the VM.
8034 * @param pVCpu Pointer to the VMCPU.
8035 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8036 * out-of-sync. Make sure to update the required fields
8037 * before using them.
8038 * @param pVmxTransient Pointer to the VMX transient structure.
8039 */
8040static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8041{
8042 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8043
8044#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8045 PGMRZDynMapFlushAutoSet(pVCpu);
8046#endif
8047
8048 /* Check force flag actions that might require us to go back to ring-3. */
8049 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8050 if (rc != VINF_SUCCESS)
8051 return rc;
8052
8053#ifndef IEM_VERIFICATION_MODE_FULL
8054 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8055 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8056 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8057 {
8058 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8059 RTGCPHYS GCPhysApicBase;
8060 GCPhysApicBase = pMixedCtx->msrApicBase;
8061 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8062
8063 /* Unalias any existing mapping. */
8064 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8065 AssertRCReturn(rc, rc);
8066
8067 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8068 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8069 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8070 AssertRCReturn(rc, rc);
8071
8072 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8073 }
8074#endif /* !IEM_VERIFICATION_MODE_FULL */
8075
8076 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
8077 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8078
8079 /*
8080 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8081 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8082 */
8083 if (TRPMHasTrap(pVCpu))
8084 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8085 else if (!pVCpu->hm.s.Event.fPending)
8086 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8087
8088 /*
8089 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8090 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8091 */
8092 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8093 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8094 {
8095 Assert(rc == VINF_EM_RESET);
8096 return rc;
8097 }
8098
8099 /*
8100 * No longjmps to ring-3 from this point on!!!
8101 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8102 * This also disables flushing of the R0-logger instance (if any).
8103 */
8104 VMMRZCallRing3Disable(pVCpu);
8105
8106 /*
8107 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8108 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8109 *
8110 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8111 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8112 *
8113 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8114 * executing guest code.
8115 */
8116 pVmxTransient->uEflags = ASMIntDisableFlags();
8117 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8118 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8119 {
8120 hmR0VmxClearEventVmcs(pVCpu);
8121 ASMSetFlags(pVmxTransient->uEflags);
8122 VMMRZCallRing3Enable(pVCpu);
8123 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8124 return VINF_EM_RAW_TO_R3;
8125 }
8126
8127 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8128 {
8129 hmR0VmxClearEventVmcs(pVCpu);
8130 ASMSetFlags(pVmxTransient->uEflags);
8131 VMMRZCallRing3Enable(pVCpu);
8132 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8133 return VINF_EM_RAW_INTERRUPT;
8134 }
8135
8136 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8137 pVCpu->hm.s.Event.fPending = false;
8138
8139 return VINF_SUCCESS;
8140}
8141
8142
8143/**
8144 * Prepares to run guest code in VT-x and we've committed to doing so. This
8145 * means there is no backing out to ring-3 or anywhere else at this
8146 * point.
8147 *
8148 * @param pVM Pointer to the VM.
8149 * @param pVCpu Pointer to the VMCPU.
8150 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8151 * out-of-sync. Make sure to update the required fields
8152 * before using them.
8153 * @param pVmxTransient Pointer to the VMX transient structure.
8154 *
8155 * @remarks Called with preemption disabled.
8156 * @remarks No-long-jump zone!!!
8157 */
8158static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8159{
8160 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8161 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8162 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8163
8164 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8165 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8166
8167 /*
8168 * If we are injecting events to a real-on-v86 mode guest, we may have to update
8169 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8170 * Reload only the necessary state, the assertion will catch if other parts of the code
8171 * change.
8172 */
8173 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8174 {
8175 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8176 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8177 }
8178
8179#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8180 if (!CPUMIsGuestFPUStateActive(pVCpu))
8181 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8182 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8183#endif
8184
8185 if ( pVCpu->hm.s.fUseGuestFpu
8186 && !CPUMIsGuestFPUStateActive(pVCpu))
8187 {
8188 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8189 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8190 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8191 }
8192
8193 /*
8194 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8195 */
8196 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8197 && pVCpu->hm.s.vmx.cMsrs > 0)
8198 {
8199 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8200 }
8201
8202 /*
8203 * Load the host state bits as we may've been preempted (only happens when
8204 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8205 */
8206 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8207 {
8208 /* This ASSUMES that pfnStartVM has been set up already. */
8209 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8210 AssertRC(rc);
8211 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8212 }
8213 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8214
8215 /*
8216 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8217 */
8218 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8219 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8220 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8221
8222 /* Store status of the shared guest-host state at the time of VM-entry. */
8223#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8224 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8225 {
8226 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8227 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8228 }
8229 else
8230#endif
8231 {
8232 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8233 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8234 }
8235 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8236
8237 /*
8238 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8239 */
8240 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8241 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8242
8243 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8244 RTCPUID idCurrentCpu = pCpu->idCpu;
8245 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8246 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8247 {
8248 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8249 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8250 }
8251
8252 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8253 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8254 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8255 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8256
8257 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8258
8259 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8260 to start executing. */
8261
8262 /*
8263 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8264 */
8265 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8266 {
8267 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8268 {
8269 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8270 AssertRC(rc2);
8271 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8272 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8273 true /* fUpdateHostMsr */);
8274 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8275 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8276 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8277 }
8278 else
8279 {
8280 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8281 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8282 }
8283 }
8284#ifdef VBOX_STRICT
8285 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8286#endif
8287}
8288
8289
8290/**
8291 * Performs some essential restoration of state after running guest code in
8292 * VT-x.
8293 *
8294 * @param pVM Pointer to the VM.
8295 * @param pVCpu Pointer to the VMCPU.
8296 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8297 * out-of-sync. Make sure to update the required fields
8298 * before using them.
8299 * @param pVmxTransient Pointer to the VMX transient structure.
8300 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8301 *
8302 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8303 *
8304 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8305 * unconditionally when it is safe to do so.
8306 */
8307static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8308{
8309 NOREF(pVM);
8310
8311 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8312
8313 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8314 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8315 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8316 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8317 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8318
8319 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8320 {
8321 /** @todo Find a way to fix hardcoding a guestimate. */
8322 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8323 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8324 }
8325
8326 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8327 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8328 Assert(!(ASMGetFlags() & X86_EFL_IF));
8329 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8330
8331#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8332 if (CPUMIsGuestFPUStateActive(pVCpu))
8333 {
8334 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8335 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8336 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8337 }
8338#endif
8339
8340#if HC_ARCH_BITS == 64
8341 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8342#endif
8343 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8344 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8345 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8346
8347 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8348 uint32_t uExitReason;
8349 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8350 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8351 AssertRC(rc);
8352 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8353 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8354
8355 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8356 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8357 {
8358 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8359 pVmxTransient->fVMEntryFailed));
8360 return;
8361 }
8362
8363 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8364 {
8365 /* Update the guest interruptibility-state from the VMCS. */
8366 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8367#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8368 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8369 AssertRC(rc);
8370#endif
8371 /*
8372 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8373 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8374 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8375 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8376 */
8377 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8378 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8379 {
8380 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8381 AssertRC(rc);
8382 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8383 }
8384 }
8385}
8386
8387
8388/**
8389 * Runs the guest code using VT-x the normal way.
8390 *
8391 * @returns VBox status code.
8392 * @param pVM Pointer to the VM.
8393 * @param pVCpu Pointer to the VMCPU.
8394 * @param pCtx Pointer to the guest-CPU context.
8395 *
8396 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8397 */
8398static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8399{
8400 VMXTRANSIENT VmxTransient;
8401 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8402 int rc = VERR_INTERNAL_ERROR_5;
8403 uint32_t cLoops = 0;
8404
8405 for (;; cLoops++)
8406 {
8407 Assert(!HMR0SuspendPending());
8408 HMVMX_ASSERT_CPU_SAFE();
8409
8410 /* Preparatory work for running guest code, this may force us to return
8411 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8412 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8413 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8414 if (rc != VINF_SUCCESS)
8415 break;
8416
8417 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8418 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8419 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8420
8421 /* Restore any residual host-state and save any bits shared between host
8422 and guest into the guest-CPU state. Re-enables interrupts! */
8423 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8424
8425 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8426 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8427 {
8428 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8429 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8430 return rc;
8431 }
8432
8433 /* Handle the VM-exit. */
8434 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8435 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8436 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8437 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8438 HMVMX_START_EXIT_DISPATCH_PROF();
8439#ifdef HMVMX_USE_FUNCTION_TABLE
8440 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8441#else
8442 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8443#endif
8444 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8445 if (rc != VINF_SUCCESS)
8446 break;
8447 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8448 {
8449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8450 rc = VINF_EM_RAW_INTERRUPT;
8451 break;
8452 }
8453 }
8454
8455 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8456 return rc;
8457}
8458
8459
8460/**
8461 * Single steps guest code using VT-x.
8462 *
8463 * @returns VBox status code.
8464 * @param pVM Pointer to the VM.
8465 * @param pVCpu Pointer to the VMCPU.
8466 * @param pCtx Pointer to the guest-CPU context.
8467 *
8468 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8469 */
8470static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8471{
8472 VMXTRANSIENT VmxTransient;
8473 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8474 int rc = VERR_INTERNAL_ERROR_5;
8475 uint32_t cLoops = 0;
8476 uint16_t uCsStart = pCtx->cs.Sel;
8477 uint64_t uRipStart = pCtx->rip;
8478
8479 for (;; cLoops++)
8480 {
8481 Assert(!HMR0SuspendPending());
8482 HMVMX_ASSERT_CPU_SAFE();
8483
8484 /* Preparatory work for running guest code, this may force us to return
8485 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8486 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8487 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8488 if (rc != VINF_SUCCESS)
8489 break;
8490
8491 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8492 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8493 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8494
8495 /* Restore any residual host-state and save any bits shared between host
8496 and guest into the guest-CPU state. Re-enables interrupts! */
8497 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8498
8499 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8500 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8501 {
8502 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8503 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8504 return rc;
8505 }
8506
8507 /* Handle the VM-exit. */
8508 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8509 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8510 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8511 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8512 HMVMX_START_EXIT_DISPATCH_PROF();
8513#ifdef HMVMX_USE_FUNCTION_TABLE
8514 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8515#else
8516 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8517#endif
8518 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8519 if (rc != VINF_SUCCESS)
8520 break;
8521 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8522 {
8523 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8524 rc = VINF_EM_RAW_INTERRUPT;
8525 break;
8526 }
8527
8528 /*
8529 * Did the RIP change, if so, consider it a single step.
8530 * Otherwise, make sure one of the TFs gets set.
8531 */
8532 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8533 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8534 AssertRCReturn(rc2, rc2);
8535 if ( pCtx->rip != uRipStart
8536 || pCtx->cs.Sel != uCsStart)
8537 {
8538 rc = VINF_EM_DBG_STEPPED;
8539 break;
8540 }
8541 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8542 }
8543
8544 /*
8545 * Clear the X86_EFL_TF if necessary.
8546 */
8547 if (pVCpu->hm.s.fClearTrapFlag)
8548 {
8549 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8550 AssertRCReturn(rc2, rc2);
8551 pVCpu->hm.s.fClearTrapFlag = false;
8552 pCtx->eflags.Bits.u1TF = 0;
8553 }
8554 /** @todo there seems to be issues with the resume flag when the monitor trap
8555 * flag is pending without being used. Seen early in bios init when
8556 * accessing APIC page in protected mode. */
8557
8558 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8559 return rc;
8560}
8561
8562
8563/**
8564 * Runs the guest code using VT-x.
8565 *
8566 * @returns VBox status code.
8567 * @param pVM Pointer to the VM.
8568 * @param pVCpu Pointer to the VMCPU.
8569 * @param pCtx Pointer to the guest-CPU context.
8570 */
8571VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8572{
8573 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8574 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8575 HMVMX_ASSERT_PREEMPT_SAFE();
8576
8577 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8578
8579 int rc;
8580 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8581 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8582 else
8583 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8584
8585 if (rc == VERR_EM_INTERPRETER)
8586 rc = VINF_EM_RAW_EMULATE_INSTR;
8587 else if (rc == VINF_EM_RESET)
8588 rc = VINF_EM_TRIPLE_FAULT;
8589
8590 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8591 if (RT_FAILURE(rc2))
8592 {
8593 pVCpu->hm.s.u32HMError = rc;
8594 rc = rc2;
8595 }
8596 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8597 return rc;
8598}
8599
8600
8601#ifndef HMVMX_USE_FUNCTION_TABLE
8602DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8603{
8604#ifdef DEBUG_ramshankar
8605# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8606# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8607#endif
8608 int rc;
8609 switch (rcReason)
8610 {
8611 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8612 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8613 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8614 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8615 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8616 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8617 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8618 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8619 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8620 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8621 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8622 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8623 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8624 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8625 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8626 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8627 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8628 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8629 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8630 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8631 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8632 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8633 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8634 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8635 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8636 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8637 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8638 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8639 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8640 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8641 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8642 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8643 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8644
8645 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8646 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8647 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8648 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8649 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8650 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8651 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8652 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8653 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8654
8655 case VMX_EXIT_VMCALL:
8656 case VMX_EXIT_VMCLEAR:
8657 case VMX_EXIT_VMLAUNCH:
8658 case VMX_EXIT_VMPTRLD:
8659 case VMX_EXIT_VMPTRST:
8660 case VMX_EXIT_VMREAD:
8661 case VMX_EXIT_VMRESUME:
8662 case VMX_EXIT_VMWRITE:
8663 case VMX_EXIT_VMXOFF:
8664 case VMX_EXIT_VMXON:
8665 case VMX_EXIT_INVEPT:
8666 case VMX_EXIT_INVVPID:
8667 case VMX_EXIT_VMFUNC:
8668 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8669 break;
8670 default:
8671 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8672 break;
8673 }
8674 return rc;
8675}
8676#endif
8677
8678#ifdef DEBUG
8679/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8680# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8681 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8682
8683# define HMVMX_ASSERT_PREEMPT_CPUID() \
8684 do \
8685 { \
8686 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8687 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8688 } while (0)
8689
8690# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8691 do { \
8692 AssertPtr(pVCpu); \
8693 AssertPtr(pMixedCtx); \
8694 AssertPtr(pVmxTransient); \
8695 Assert(pVmxTransient->fVMEntryFailed == false); \
8696 Assert(ASMIntAreEnabled()); \
8697 HMVMX_ASSERT_PREEMPT_SAFE(); \
8698 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8699 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8700 HMVMX_ASSERT_PREEMPT_SAFE(); \
8701 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8702 HMVMX_ASSERT_PREEMPT_CPUID(); \
8703 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8704 } while (0)
8705
8706# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8707 do { \
8708 Log4Func(("\n")); \
8709 } while (0)
8710#else /* Release builds */
8711# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8712 do { \
8713 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8714 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8715 } while (0)
8716# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8717#endif
8718
8719
8720/**
8721 * Advances the guest RIP after reading it from the VMCS.
8722 *
8723 * @returns VBox status code.
8724 * @param pVCpu Pointer to the VMCPU.
8725 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8726 * out-of-sync. Make sure to update the required fields
8727 * before using them.
8728 * @param pVmxTransient Pointer to the VMX transient structure.
8729 *
8730 * @remarks No-long-jump zone!!!
8731 */
8732DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8733{
8734 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8735 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8736 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8737 AssertRCReturn(rc, rc);
8738
8739 pMixedCtx->rip += pVmxTransient->cbInstr;
8740 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8741
8742 /*
8743 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8744 * pending debug exception field as it takes care of priority of events.
8745 *
8746 * See Intel spec. 32.2.1 "Debug Exceptions".
8747 */
8748 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8749
8750 return rc;
8751}
8752
8753
8754/**
8755 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8756 * and update error record fields accordingly.
8757 *
8758 * @return VMX_IGS_* return codes.
8759 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8760 * wrong with the guest state.
8761 *
8762 * @param pVM Pointer to the VM.
8763 * @param pVCpu Pointer to the VMCPU.
8764 * @param pCtx Pointer to the guest-CPU state.
8765 */
8766static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8767{
8768#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8769#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8770 uError = (err); \
8771 break; \
8772 } else do { } while (0)
8773
8774 int rc;
8775 uint32_t uError = VMX_IGS_ERROR;
8776 uint32_t u32Val;
8777 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8778
8779 do
8780 {
8781 /*
8782 * CR0.
8783 */
8784 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8785 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8786 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8787 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8788 if (fUnrestrictedGuest)
8789 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8790
8791 uint32_t u32GuestCR0;
8792 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8793 AssertRCBreak(rc);
8794 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8795 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8796 if ( !fUnrestrictedGuest
8797 && (u32GuestCR0 & X86_CR0_PG)
8798 && !(u32GuestCR0 & X86_CR0_PE))
8799 {
8800 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8801 }
8802
8803 /*
8804 * CR4.
8805 */
8806 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8807 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8808
8809 uint32_t u32GuestCR4;
8810 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8811 AssertRCBreak(rc);
8812 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8813 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8814
8815 /*
8816 * IA32_DEBUGCTL MSR.
8817 */
8818 uint64_t u64Val;
8819 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8820 AssertRCBreak(rc);
8821 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8822 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8823 {
8824 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8825 }
8826 uint64_t u64DebugCtlMsr = u64Val;
8827
8828#ifdef VBOX_STRICT
8829 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8830 AssertRCBreak(rc);
8831 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8832#endif
8833 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8834
8835 /*
8836 * RIP and RFLAGS.
8837 */
8838 uint32_t u32Eflags;
8839#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8840 if (HMVMX_IS_64BIT_HOST_MODE())
8841 {
8842 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8843 AssertRCBreak(rc);
8844 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8845 if ( !fLongModeGuest
8846 || !pCtx->cs.Attr.n.u1Long)
8847 {
8848 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8849 }
8850 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8851 * must be identical if the "IA32e mode guest" VM-entry control is 1
8852 * and CS.L is 1. No check applies if the CPU supports 64
8853 * linear-address bits. */
8854
8855 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8856 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8857 AssertRCBreak(rc);
8858 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8859 VMX_IGS_RFLAGS_RESERVED);
8860 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8861 u32Eflags = u64Val;
8862 }
8863 else
8864#endif
8865 {
8866 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8867 AssertRCBreak(rc);
8868 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8869 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8870 }
8871
8872 if ( fLongModeGuest
8873 || ( fUnrestrictedGuest
8874 && !(u32GuestCR0 & X86_CR0_PE)))
8875 {
8876 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8877 }
8878
8879 uint32_t u32EntryInfo;
8880 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8881 AssertRCBreak(rc);
8882 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8883 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8884 {
8885 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8886 }
8887
8888 /*
8889 * 64-bit checks.
8890 */
8891#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8892 if (HMVMX_IS_64BIT_HOST_MODE())
8893 {
8894 if ( fLongModeGuest
8895 && !fUnrestrictedGuest)
8896 {
8897 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8898 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8899 }
8900
8901 if ( !fLongModeGuest
8902 && (u32GuestCR4 & X86_CR4_PCIDE))
8903 {
8904 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8905 }
8906
8907 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8908 * 51:32 beyond the processor's physical-address width are 0. */
8909
8910 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8911 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8912 {
8913 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8914 }
8915
8916 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8917 AssertRCBreak(rc);
8918 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8919
8920 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8921 AssertRCBreak(rc);
8922 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8923 }
8924#endif
8925
8926 /*
8927 * PERF_GLOBAL MSR.
8928 */
8929 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8930 {
8931 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8932 AssertRCBreak(rc);
8933 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8934 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8935 }
8936
8937 /*
8938 * PAT MSR.
8939 */
8940 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8941 {
8942 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8943 AssertRCBreak(rc);
8944 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8945 for (unsigned i = 0; i < 8; i++)
8946 {
8947 uint8_t u8Val = (u64Val & 0x7);
8948 if ( u8Val != 0 /* UC */
8949 || u8Val != 1 /* WC */
8950 || u8Val != 4 /* WT */
8951 || u8Val != 5 /* WP */
8952 || u8Val != 6 /* WB */
8953 || u8Val != 7 /* UC- */)
8954 {
8955 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8956 }
8957 u64Val >>= 3;
8958 }
8959 }
8960
8961 /*
8962 * EFER MSR.
8963 */
8964 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8965 {
8966 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8967 AssertRCBreak(rc);
8968 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8969 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8970 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8971 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8972 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8973 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8974 }
8975
8976 /*
8977 * Segment registers.
8978 */
8979 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8980 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8981 if (!(u32Eflags & X86_EFL_VM))
8982 {
8983 /* CS */
8984 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8985 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8986 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8987 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8988 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8989 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8990 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8991 /* CS cannot be loaded with NULL in protected mode. */
8992 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8993 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8994 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8995 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8996 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8997 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8998 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8999 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9000 else
9001 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9002
9003 /* SS */
9004 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9005 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9006 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9007 if ( !(pCtx->cr0 & X86_CR0_PE)
9008 || pCtx->cs.Attr.n.u4Type == 3)
9009 {
9010 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9011 }
9012 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9013 {
9014 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9015 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9016 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9017 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9018 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9019 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9020 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9021 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9022 }
9023
9024 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9025 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9026 {
9027 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9028 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9029 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9030 || pCtx->ds.Attr.n.u4Type > 11
9031 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9032 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9033 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9034 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9035 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9036 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9037 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9038 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9039 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9040 }
9041 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9042 {
9043 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9044 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9045 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9046 || pCtx->es.Attr.n.u4Type > 11
9047 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9048 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9049 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9050 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9051 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9052 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9053 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9054 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9055 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9056 }
9057 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9058 {
9059 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9060 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9061 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9062 || pCtx->fs.Attr.n.u4Type > 11
9063 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9064 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9065 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9066 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9067 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9068 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9069 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9070 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9071 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9072 }
9073 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9074 {
9075 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9076 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9077 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9078 || pCtx->gs.Attr.n.u4Type > 11
9079 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9080 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9081 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9082 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9083 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9084 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9085 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9086 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9087 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9088 }
9089 /* 64-bit capable CPUs. */
9090#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9091 if (HMVMX_IS_64BIT_HOST_MODE())
9092 {
9093 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9094 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9095 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9096 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9097 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9098 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9099 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9100 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9101 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9102 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9103 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9104 }
9105#endif
9106 }
9107 else
9108 {
9109 /* V86 mode checks. */
9110 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9111 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9112 {
9113 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9114 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9115 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9116 }
9117 else
9118 {
9119 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9120 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9121 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9122 }
9123
9124 /* CS */
9125 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9126 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9127 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9128 /* SS */
9129 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9130 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9131 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9132 /* DS */
9133 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9134 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9135 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9136 /* ES */
9137 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9138 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9139 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9140 /* FS */
9141 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9142 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9143 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9144 /* GS */
9145 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9146 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9147 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9148 /* 64-bit capable CPUs. */
9149#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9150 if (HMVMX_IS_64BIT_HOST_MODE())
9151 {
9152 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9153 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9154 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9155 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9156 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9157 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9158 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9159 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9160 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9161 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9162 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9163 }
9164#endif
9165 }
9166
9167 /*
9168 * TR.
9169 */
9170 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9171 /* 64-bit capable CPUs. */
9172#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9173 if (HMVMX_IS_64BIT_HOST_MODE())
9174 {
9175 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9176 }
9177#endif
9178 if (fLongModeGuest)
9179 {
9180 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9181 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9182 }
9183 else
9184 {
9185 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9186 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9187 VMX_IGS_TR_ATTR_TYPE_INVALID);
9188 }
9189 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9190 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9191 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9192 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9193 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9194 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9195 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9196 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9197
9198 /*
9199 * GDTR and IDTR.
9200 */
9201#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9202 if (HMVMX_IS_64BIT_HOST_MODE())
9203 {
9204 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9205 AssertRCBreak(rc);
9206 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9207
9208 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9209 AssertRCBreak(rc);
9210 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9211 }
9212#endif
9213
9214 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9215 AssertRCBreak(rc);
9216 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9217
9218 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9219 AssertRCBreak(rc);
9220 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9221
9222 /*
9223 * Guest Non-Register State.
9224 */
9225 /* Activity State. */
9226 uint32_t u32ActivityState;
9227 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9228 AssertRCBreak(rc);
9229 HMVMX_CHECK_BREAK( !u32ActivityState
9230 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9231 VMX_IGS_ACTIVITY_STATE_INVALID);
9232 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9233 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9234 uint32_t u32IntrState;
9235 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9236 AssertRCBreak(rc);
9237 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9238 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9239 {
9240 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9241 }
9242
9243 /** @todo Activity state and injecting interrupts. Left as a todo since we
9244 * currently don't use activity states but ACTIVE. */
9245
9246 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9247 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9248
9249 /* Guest interruptibility-state. */
9250 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9251 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9252 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9253 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9254 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9255 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9256 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9257 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9258 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9259 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9260 {
9261 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9262 {
9263 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9264 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9265 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9266 }
9267 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9268 {
9269 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9270 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9271 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9272 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9273 }
9274 }
9275 /** @todo Assumes the processor is not in SMM. */
9276 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9277 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9278 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9279 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9280 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9281 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9282 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9283 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9284 {
9285 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9286 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9287 }
9288
9289 /* Pending debug exceptions. */
9290 if (HMVMX_IS_64BIT_HOST_MODE())
9291 {
9292 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9293 AssertRCBreak(rc);
9294 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9295 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9296 u32Val = u64Val; /* For pending debug exceptions checks below. */
9297 }
9298 else
9299 {
9300 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9301 AssertRCBreak(rc);
9302 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9303 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9304 }
9305
9306 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9307 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9308 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9309 {
9310 if ( (u32Eflags & X86_EFL_TF)
9311 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9312 {
9313 /* Bit 14 is PendingDebug.BS. */
9314 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9315 }
9316 if ( !(u32Eflags & X86_EFL_TF)
9317 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9318 {
9319 /* Bit 14 is PendingDebug.BS. */
9320 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9321 }
9322 }
9323
9324 /* VMCS link pointer. */
9325 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9326 AssertRCBreak(rc);
9327 if (u64Val != UINT64_C(0xffffffffffffffff))
9328 {
9329 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9330 /** @todo Bits beyond the processor's physical-address width MBZ. */
9331 /** @todo 32-bit located in memory referenced by value of this field (as a
9332 * physical address) must contain the processor's VMCS revision ID. */
9333 /** @todo SMM checks. */
9334 }
9335
9336 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9337 * not using Nested Paging? */
9338 if ( pVM->hm.s.fNestedPaging
9339 && !fLongModeGuest
9340 && CPUMIsGuestInPAEModeEx(pCtx))
9341 {
9342 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9343 AssertRCBreak(rc);
9344 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9345
9346 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9347 AssertRCBreak(rc);
9348 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9349
9350 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9351 AssertRCBreak(rc);
9352 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9353
9354 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9355 AssertRCBreak(rc);
9356 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9357 }
9358
9359 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9360 if (uError == VMX_IGS_ERROR)
9361 uError = VMX_IGS_REASON_NOT_FOUND;
9362 } while (0);
9363
9364 pVCpu->hm.s.u32HMError = uError;
9365 return uError;
9366
9367#undef HMVMX_ERROR_BREAK
9368#undef HMVMX_CHECK_BREAK
9369}
9370
9371/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9372/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9373/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9374
9375/** @name VM-exit handlers.
9376 * @{
9377 */
9378
9379/**
9380 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9381 */
9382HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9383{
9384 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9385 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9386 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9387 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9388 return VINF_SUCCESS;
9389 return VINF_EM_RAW_INTERRUPT;
9390}
9391
9392
9393/**
9394 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9395 */
9396HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9397{
9398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9399 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9400
9401 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9402 AssertRCReturn(rc, rc);
9403
9404 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9405 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9406 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9407 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9408
9409 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9410 {
9411 /*
9412 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9413 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9414 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9415 *
9416 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9417 */
9418 VMXDispatchHostNmi();
9419 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9420 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9421 return VINF_SUCCESS;
9422 }
9423
9424 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9425 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9426 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9427 {
9428 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9429 return VINF_SUCCESS;
9430 }
9431 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9432 {
9433 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9434 return rc;
9435 }
9436
9437 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9438 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9439 switch (uIntType)
9440 {
9441 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9442 Assert(uVector == X86_XCPT_DB);
9443 /* no break */
9444 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9445 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9446 /* no break */
9447 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9448 {
9449 switch (uVector)
9450 {
9451 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9452 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9453 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9454 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9455 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9456 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9457#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9458 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9459 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9460 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9461 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9462 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9463 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9464 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9465 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9466 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9467 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9468 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9469 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9470#endif
9471 default:
9472 {
9473 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9474 AssertRCReturn(rc, rc);
9475
9476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9477 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9478 {
9479 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9480 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9481 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9482
9483 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9484 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9485 AssertRCReturn(rc, rc);
9486 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9487 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9488 0 /* GCPtrFaultAddress */);
9489 AssertRCReturn(rc, rc);
9490 }
9491 else
9492 {
9493 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9494 pVCpu->hm.s.u32HMError = uVector;
9495 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9496 }
9497 break;
9498 }
9499 }
9500 break;
9501 }
9502
9503 default:
9504 {
9505 pVCpu->hm.s.u32HMError = uExitIntInfo;
9506 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9507 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9508 break;
9509 }
9510 }
9511 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9512 return rc;
9513}
9514
9515
9516/**
9517 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9518 */
9519HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9520{
9521 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9522
9523 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9524 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9525
9526 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9527 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9528 return VINF_SUCCESS;
9529}
9530
9531
9532/**
9533 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9534 */
9535HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9536{
9537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9538 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9539 HMVMX_RETURN_UNEXPECTED_EXIT();
9540}
9541
9542
9543/**
9544 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9545 */
9546HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9547{
9548 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9549 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9550 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9551}
9552
9553
9554/**
9555 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9556 */
9557HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9558{
9559 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9560 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9561 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9562}
9563
9564
9565/**
9566 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9567 */
9568HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9569{
9570 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9571 PVM pVM = pVCpu->CTX_SUFF(pVM);
9572 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9573 if (RT_LIKELY(rc == VINF_SUCCESS))
9574 {
9575 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9576 Assert(pVmxTransient->cbInstr == 2);
9577 }
9578 else
9579 {
9580 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9581 rc = VERR_EM_INTERPRETER;
9582 }
9583 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9584 return rc;
9585}
9586
9587
9588/**
9589 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9590 */
9591HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9592{
9593 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9594 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9595 AssertRCReturn(rc, rc);
9596
9597 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9598 return VINF_EM_RAW_EMULATE_INSTR;
9599
9600 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9601 HMVMX_RETURN_UNEXPECTED_EXIT();
9602}
9603
9604
9605/**
9606 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9607 */
9608HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9609{
9610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9611 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9612 AssertRCReturn(rc, rc);
9613
9614 PVM pVM = pVCpu->CTX_SUFF(pVM);
9615 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9616 if (RT_LIKELY(rc == VINF_SUCCESS))
9617 {
9618 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9619 Assert(pVmxTransient->cbInstr == 2);
9620 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9621 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9622 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9623 }
9624 else
9625 {
9626 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9627 rc = VERR_EM_INTERPRETER;
9628 }
9629 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9630 return rc;
9631}
9632
9633
9634/**
9635 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9636 */
9637HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9638{
9639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9640 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9641 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9642 AssertRCReturn(rc, rc);
9643
9644 PVM pVM = pVCpu->CTX_SUFF(pVM);
9645 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9646 if (RT_LIKELY(rc == VINF_SUCCESS))
9647 {
9648 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9649 Assert(pVmxTransient->cbInstr == 3);
9650 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9651 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9652 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9653 }
9654 else
9655 {
9656 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9657 rc = VERR_EM_INTERPRETER;
9658 }
9659 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9660 return rc;
9661}
9662
9663
9664/**
9665 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9666 */
9667HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9668{
9669 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9670 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9671 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9672 AssertRCReturn(rc, rc);
9673
9674 PVM pVM = pVCpu->CTX_SUFF(pVM);
9675 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9676 if (RT_LIKELY(rc == VINF_SUCCESS))
9677 {
9678 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9679 Assert(pVmxTransient->cbInstr == 2);
9680 }
9681 else
9682 {
9683 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9684 rc = VERR_EM_INTERPRETER;
9685 }
9686 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9687 return rc;
9688}
9689
9690
9691/**
9692 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9693 */
9694HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9695{
9696 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9697 PVM pVM = pVCpu->CTX_SUFF(pVM);
9698 Assert(!pVM->hm.s.fNestedPaging);
9699
9700 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9701 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9702 AssertRCReturn(rc, rc);
9703
9704 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9705 rc = VBOXSTRICTRC_VAL(rc2);
9706 if (RT_LIKELY(rc == VINF_SUCCESS))
9707 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9708 else
9709 {
9710 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9711 pVmxTransient->uExitQualification, rc));
9712 }
9713 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9714 return rc;
9715}
9716
9717
9718/**
9719 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9720 */
9721HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9722{
9723 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9724 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9725 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9726 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9727 AssertRCReturn(rc, rc);
9728
9729 PVM pVM = pVCpu->CTX_SUFF(pVM);
9730 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9731 if (RT_LIKELY(rc == VINF_SUCCESS))
9732 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9733 else
9734 {
9735 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9736 rc = VERR_EM_INTERPRETER;
9737 }
9738 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9739 return rc;
9740}
9741
9742
9743/**
9744 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9745 */
9746HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9747{
9748 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9749 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9750 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9751 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9752 AssertRCReturn(rc, rc);
9753
9754 PVM pVM = pVCpu->CTX_SUFF(pVM);
9755 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9756 rc = VBOXSTRICTRC_VAL(rc2);
9757 if (RT_LIKELY( rc == VINF_SUCCESS
9758 || rc == VINF_EM_HALT))
9759 {
9760 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9761 AssertRCReturn(rc3, rc3);
9762
9763 if ( rc == VINF_EM_HALT
9764 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9765 {
9766 rc = VINF_SUCCESS;
9767 }
9768 }
9769 else
9770 {
9771 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9772 rc = VERR_EM_INTERPRETER;
9773 }
9774 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9775 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9776 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9777 return rc;
9778}
9779
9780
9781/**
9782 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9783 */
9784HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9785{
9786 /*
9787 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9788 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9789 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9790 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9791 */
9792 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9793 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9794 HMVMX_RETURN_UNEXPECTED_EXIT();
9795}
9796
9797
9798/**
9799 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9800 */
9801HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9802{
9803 /*
9804 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9805 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9806 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9807 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9808 */
9809 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9810 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9811 HMVMX_RETURN_UNEXPECTED_EXIT();
9812}
9813
9814
9815/**
9816 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9817 */
9818HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819{
9820 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9821 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9822 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9823 HMVMX_RETURN_UNEXPECTED_EXIT();
9824}
9825
9826
9827/**
9828 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9829 */
9830HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9831{
9832 /*
9833 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9834 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9835 * See Intel spec. 25.3 "Other Causes of VM-exits".
9836 */
9837 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9838 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9839 HMVMX_RETURN_UNEXPECTED_EXIT();
9840}
9841
9842
9843/**
9844 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9845 * VM-exit.
9846 */
9847HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9848{
9849 /*
9850 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9851 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9852 *
9853 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9854 * See Intel spec. "23.8 Restrictions on VMX operation".
9855 */
9856 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9857 return VINF_SUCCESS;
9858}
9859
9860
9861/**
9862 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9863 * VM-exit.
9864 */
9865HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9866{
9867 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9868 return VINF_EM_RESET;
9869}
9870
9871
9872/**
9873 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9874 */
9875HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9876{
9877 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9878 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9879 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9880 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9881 AssertRCReturn(rc, rc);
9882
9883 pMixedCtx->rip++;
9884 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9885 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9886 rc = VINF_SUCCESS;
9887 else
9888 rc = VINF_EM_HALT;
9889
9890 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9891 return rc;
9892}
9893
9894
9895/**
9896 * VM-exit handler for instructions that result in a #UD exception delivered to
9897 * the guest.
9898 */
9899HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9900{
9901 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9902 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9903 return VINF_SUCCESS;
9904}
9905
9906
9907/**
9908 * VM-exit handler for expiry of the VMX preemption timer.
9909 */
9910HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9911{
9912 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9913
9914 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9915 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9916
9917 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9918 PVM pVM = pVCpu->CTX_SUFF(pVM);
9919 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9920 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9921 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9922}
9923
9924
9925/**
9926 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9927 */
9928HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9929{
9930 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9931
9932 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9933 /** @todo check if XSETBV is supported by the recompiler. */
9934 return VERR_EM_INTERPRETER;
9935}
9936
9937
9938/**
9939 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9940 */
9941HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9942{
9943 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9944
9945 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9946 /** @todo implement EMInterpretInvpcid() */
9947 return VERR_EM_INTERPRETER;
9948}
9949
9950
9951/**
9952 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9953 * Error VM-exit.
9954 */
9955HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9956{
9957 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9958 AssertRCReturn(rc, rc);
9959
9960 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9961 NOREF(uInvalidReason);
9962
9963#ifdef VBOX_STRICT
9964 uint32_t uIntrState;
9965 HMVMXHCUINTREG uHCReg;
9966 uint64_t u64Val;
9967 uint32_t u32Val;
9968
9969 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9970 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9971 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9972 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9973 AssertRCReturn(rc, rc);
9974
9975 Log4(("uInvalidReason %u\n", uInvalidReason));
9976 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9977 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9978 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9979 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9980
9981 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9982 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9983 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9984 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9985 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9986 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9987 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9988 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9989 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9990 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9991 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9992 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9993#else
9994 NOREF(pVmxTransient);
9995#endif
9996
9997 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9998 return VERR_VMX_INVALID_GUEST_STATE;
9999}
10000
10001
10002/**
10003 * VM-exit handler for VM-entry failure due to an MSR-load
10004 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10005 */
10006HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10007{
10008 NOREF(pVmxTransient);
10009 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10010 HMVMX_RETURN_UNEXPECTED_EXIT();
10011}
10012
10013
10014/**
10015 * VM-exit handler for VM-entry failure due to a machine-check event
10016 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10017 */
10018HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10019{
10020 NOREF(pVmxTransient);
10021 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10022 HMVMX_RETURN_UNEXPECTED_EXIT();
10023}
10024
10025
10026/**
10027 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10028 * theory.
10029 */
10030HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10031{
10032 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10033 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10034 return VERR_VMX_UNDEFINED_EXIT_CODE;
10035}
10036
10037
10038/**
10039 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10040 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10041 * Conditional VM-exit.
10042 */
10043HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10044{
10045 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10046
10047 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10048 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10049 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10050 return VERR_EM_INTERPRETER;
10051 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10052 HMVMX_RETURN_UNEXPECTED_EXIT();
10053}
10054
10055
10056/**
10057 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10058 */
10059HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10060{
10061 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10062
10063 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10064 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10065 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10066 return VERR_EM_INTERPRETER;
10067 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10068 HMVMX_RETURN_UNEXPECTED_EXIT();
10069}
10070
10071
10072/**
10073 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10074 */
10075HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10076{
10077 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10078
10079 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10080 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10081 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10082 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10083 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10084 {
10085 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10086 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10087 }
10088 AssertRCReturn(rc, rc);
10089 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10090
10091#ifdef VBOX_STRICT
10092 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10093 {
10094 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10095 {
10096 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10097 HMVMX_RETURN_UNEXPECTED_EXIT();
10098 }
10099# if HC_ARCH_BITS == 64
10100 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10101 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10102 {
10103 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10104 HMVMX_RETURN_UNEXPECTED_EXIT();
10105 }
10106# endif
10107 }
10108#endif
10109
10110 PVM pVM = pVCpu->CTX_SUFF(pVM);
10111 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10112 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10113 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10114 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10115
10116 if (RT_LIKELY(rc == VINF_SUCCESS))
10117 {
10118 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10119 Assert(pVmxTransient->cbInstr == 2);
10120 }
10121 return rc;
10122}
10123
10124
10125/**
10126 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10127 */
10128HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10129{
10130 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10131 PVM pVM = pVCpu->CTX_SUFF(pVM);
10132 int rc = VINF_SUCCESS;
10133
10134 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10135 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10136 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10137 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10138 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10139 {
10140 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10141 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10142 }
10143 AssertRCReturn(rc, rc);
10144 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10145
10146 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10147 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10149
10150 if (RT_LIKELY(rc == VINF_SUCCESS))
10151 {
10152 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10153
10154 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10155 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10156 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10157 {
10158 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10159 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10160 EMInterpretWrmsr() changes it. */
10161 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10162 }
10163 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10164 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10165
10166 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10167 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10168 {
10169 switch (pMixedCtx->ecx)
10170 {
10171 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10172 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10173 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10174 case MSR_K8_FS_BASE: /* no break */
10175 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10176 default:
10177 {
10178 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10179 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10180#if HC_ARCH_BITS == 64
10181 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10182 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10183#endif
10184 break;
10185 }
10186 }
10187 }
10188#ifdef VBOX_STRICT
10189 else
10190 {
10191 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10192 switch (pMixedCtx->ecx)
10193 {
10194 case MSR_IA32_SYSENTER_CS:
10195 case MSR_IA32_SYSENTER_EIP:
10196 case MSR_IA32_SYSENTER_ESP:
10197 case MSR_K8_FS_BASE:
10198 case MSR_K8_GS_BASE:
10199 {
10200 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10201 HMVMX_RETURN_UNEXPECTED_EXIT();
10202 }
10203
10204 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10205 default:
10206 {
10207 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10208 {
10209 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10210 pMixedCtx->ecx));
10211 HMVMX_RETURN_UNEXPECTED_EXIT();
10212 }
10213
10214#if HC_ARCH_BITS == 64
10215 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10216 {
10217 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10218 HMVMX_RETURN_UNEXPECTED_EXIT();
10219 }
10220#endif
10221 break;
10222 }
10223 }
10224 }
10225#endif /* VBOX_STRICT */
10226 }
10227 return rc;
10228}
10229
10230
10231/**
10232 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10233 */
10234HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10235{
10236 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10237
10238 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10239 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10240 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10241 return VERR_EM_INTERPRETER;
10242 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10243 HMVMX_RETURN_UNEXPECTED_EXIT();
10244}
10245
10246
10247/**
10248 * VM-exit handler for when the TPR value is lowered below the specified
10249 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10250 */
10251HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10252{
10253 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10254 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10255
10256 /*
10257 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10258 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10259 * resume guest execution.
10260 */
10261 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10262 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10263 return VINF_SUCCESS;
10264}
10265
10266
10267/**
10268 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10269 * VM-exit.
10270 *
10271 * @retval VINF_SUCCESS when guest execution can continue.
10272 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10273 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10274 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10275 * recompiler.
10276 */
10277HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10278{
10279 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10280 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10281 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10282 AssertRCReturn(rc, rc);
10283
10284 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10285 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10286 PVM pVM = pVCpu->CTX_SUFF(pVM);
10287 switch (uAccessType)
10288 {
10289 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10290 {
10291#if 0
10292 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10293 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10294#else
10295 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10296 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10297 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10298#endif
10299 AssertRCReturn(rc, rc);
10300
10301 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10302 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10303 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10304 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10305
10306 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10307 {
10308 case 0: /* CR0 */
10309 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10310 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10311 break;
10312 case 2: /* CR2 */
10313 /* Nothing to do here, CR2 it's not part of the VMCS. */
10314 break;
10315 case 3: /* CR3 */
10316 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10317 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10318 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10319 break;
10320 case 4: /* CR4 */
10321 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10322 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10323 break;
10324 case 8: /* CR8 */
10325 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10326 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10327 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10328 break;
10329 default:
10330 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10331 break;
10332 }
10333
10334 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10335 break;
10336 }
10337
10338 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10339 {
10340 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10341 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10342 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10343 AssertRCReturn(rc, rc);
10344 Assert( !pVM->hm.s.fNestedPaging
10345 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10346 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10347
10348 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10349 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10350 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10351
10352 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10353 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10354 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10355 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10357 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10358 break;
10359 }
10360
10361 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10362 {
10363 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10364 AssertRCReturn(rc, rc);
10365 rc = EMInterpretCLTS(pVM, pVCpu);
10366 AssertRCReturn(rc, rc);
10367 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10369 Log4(("CRX CLTS write rc=%d\n", rc));
10370 break;
10371 }
10372
10373 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10374 {
10375 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10376 AssertRCReturn(rc, rc);
10377 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10378 if (RT_LIKELY(rc == VINF_SUCCESS))
10379 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10380 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10381 Log4(("CRX LMSW write rc=%d\n", rc));
10382 break;
10383 }
10384
10385 default:
10386 {
10387 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10388 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10389 }
10390 }
10391
10392 /* Validate possible error codes. */
10393 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10394 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10395 if (RT_SUCCESS(rc))
10396 {
10397 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10398 AssertRCReturn(rc2, rc2);
10399 }
10400
10401 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10402 return rc;
10403}
10404
10405
10406/**
10407 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10408 * VM-exit.
10409 */
10410HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10411{
10412 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10413 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10414
10415 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10416 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10417 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10418 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10419 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10420 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10421 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10422 AssertRCReturn(rc2, rc2);
10423
10424 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10425 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10426 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10427 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10428 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10429 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10430 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10431 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10432
10433 /* I/O operation lookup arrays. */
10434 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10435 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10436
10437 VBOXSTRICTRC rcStrict;
10438 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10439 const uint32_t cbInstr = pVmxTransient->cbInstr;
10440 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10441 PVM pVM = pVCpu->CTX_SUFF(pVM);
10442 if (fIOString)
10443 {
10444#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10445 /*
10446 * INS/OUTS - I/O String instruction.
10447 *
10448 * Use instruction-information if available, otherwise fall back on
10449 * interpreting the instruction.
10450 */
10451 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10452 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10453 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10454 {
10455 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10456 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10457 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10458 AssertRCReturn(rc2, rc2);
10459 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10460 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10461 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10462 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10463 if (fIOWrite)
10464 {
10465 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10466 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10467 }
10468 else
10469 {
10470 /*
10471 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10472 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10473 * See Intel Instruction spec. for "INS".
10474 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10475 */
10476 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10477 }
10478 }
10479 else
10480 {
10481 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10482 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10483 AssertRCReturn(rc2, rc2);
10484 rcStrict = IEMExecOne(pVCpu);
10485 }
10486 /** @todo IEM needs to be setting these flags somehow. */
10487 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10488 fUpdateRipAlready = true;
10489#else
10490 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10491 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10492 if (RT_SUCCESS(rcStrict))
10493 {
10494 if (fIOWrite)
10495 {
10496 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10497 (DISCPUMODE)pDis->uAddrMode, cbValue);
10498 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10499 }
10500 else
10501 {
10502 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10503 (DISCPUMODE)pDis->uAddrMode, cbValue);
10504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10505 }
10506 }
10507 else
10508 {
10509 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10510 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10511 }
10512#endif
10513 }
10514 else
10515 {
10516 /*
10517 * IN/OUT - I/O instruction.
10518 */
10519 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10520 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10521 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10522 if (fIOWrite)
10523 {
10524 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10525 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10526 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10527 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10528 }
10529 else
10530 {
10531 uint32_t u32Result = 0;
10532 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10533 if (IOM_SUCCESS(rcStrict))
10534 {
10535 /* Save result of I/O IN instr. in AL/AX/EAX. */
10536 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10537 }
10538 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10539 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10540 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10541 }
10542 }
10543
10544 if (IOM_SUCCESS(rcStrict))
10545 {
10546 if (!fUpdateRipAlready)
10547 {
10548 pMixedCtx->rip += cbInstr;
10549 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10550 }
10551
10552 /*
10553 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10554 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10555 */
10556 if (fIOString)
10557 {
10558 /** @todo Single-step for INS/OUTS with REP prefix? */
10559 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10560 }
10561 else if (fStepping)
10562 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10563
10564 /*
10565 * If any I/O breakpoints are armed, we need to check if one triggered
10566 * and take appropriate action.
10567 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10568 */
10569 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10570 AssertRCReturn(rc2, rc2);
10571
10572 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10573 * execution engines about whether hyper BPs and such are pending. */
10574 uint32_t const uDr7 = pMixedCtx->dr[7];
10575 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10576 && X86_DR7_ANY_RW_IO(uDr7)
10577 && (pMixedCtx->cr4 & X86_CR4_DE))
10578 || DBGFBpIsHwIoArmed(pVM)))
10579 {
10580 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10581
10582 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10583 VMMRZCallRing3Disable(pVCpu);
10584 HM_DISABLE_PREEMPT_IF_NEEDED();
10585
10586 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10587
10588 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10589 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10590 {
10591 /* Raise #DB. */
10592 if (fIsGuestDbgActive)
10593 ASMSetDR6(pMixedCtx->dr[6]);
10594 if (pMixedCtx->dr[7] != uDr7)
10595 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10596
10597 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10598 }
10599 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10600 else if ( rcStrict2 != VINF_SUCCESS
10601 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10602 rcStrict = rcStrict2;
10603
10604 HM_RESTORE_PREEMPT_IF_NEEDED();
10605 VMMRZCallRing3Enable(pVCpu);
10606 }
10607 }
10608
10609#ifdef DEBUG
10610 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10611 Assert(!fIOWrite);
10612 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10613 Assert(fIOWrite);
10614 else
10615 {
10616 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10617 * statuses, that the VMM device and some others may return. See
10618 * IOM_SUCCESS() for guidance. */
10619 AssertMsg( RT_FAILURE(rcStrict)
10620 || rcStrict == VINF_SUCCESS
10621 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10622 || rcStrict == VINF_EM_DBG_BREAKPOINT
10623 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10624 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10625 }
10626#endif
10627
10628 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10629 return VBOXSTRICTRC_TODO(rcStrict);
10630}
10631
10632
10633/**
10634 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10635 * VM-exit.
10636 */
10637HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10638{
10639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10640
10641 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10642 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10643 AssertRCReturn(rc, rc);
10644 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10645 {
10646 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10647 AssertRCReturn(rc, rc);
10648 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10649 {
10650 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10651
10652 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10653 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10654
10655 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10656 Assert(!pVCpu->hm.s.Event.fPending);
10657 pVCpu->hm.s.Event.fPending = true;
10658 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10659 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10660 AssertRCReturn(rc, rc);
10661 if (fErrorCodeValid)
10662 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10663 else
10664 pVCpu->hm.s.Event.u32ErrCode = 0;
10665 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10666 && uVector == X86_XCPT_PF)
10667 {
10668 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10669 }
10670
10671 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10673 return VINF_EM_RAW_INJECT_TRPM_EVENT;
10674 }
10675 }
10676
10677 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10678 * emulation. */
10679 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10680 return VERR_EM_INTERPRETER;
10681}
10682
10683
10684/**
10685 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10686 */
10687HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10688{
10689 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10690 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10691 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10692 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10693 AssertRCReturn(rc, rc);
10694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10695 return VINF_EM_DBG_STEPPED;
10696}
10697
10698
10699/**
10700 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10701 */
10702HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10703{
10704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10705
10706 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10707 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10708 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10709 return VINF_SUCCESS;
10710 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10711 return rc;
10712
10713#if 0
10714 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10715 * just sync the whole thing. */
10716 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10717#else
10718 /* Aggressive state sync. for now. */
10719 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10720 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10721 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10722#endif
10723 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10724 AssertRCReturn(rc, rc);
10725
10726 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10727 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10728 switch (uAccessType)
10729 {
10730 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10731 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10732 {
10733 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10734 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10735 {
10736 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10737 }
10738
10739 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10740 GCPhys &= PAGE_BASE_GC_MASK;
10741 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10742 PVM pVM = pVCpu->CTX_SUFF(pVM);
10743 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10744 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10745
10746 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10747 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10748 CPUMCTX2CORE(pMixedCtx), GCPhys);
10749 rc = VBOXSTRICTRC_VAL(rc2);
10750 Log4(("ApicAccess rc=%d\n", rc));
10751 if ( rc == VINF_SUCCESS
10752 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10753 || rc == VERR_PAGE_NOT_PRESENT)
10754 {
10755 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10756 | HM_CHANGED_GUEST_RSP
10757 | HM_CHANGED_GUEST_RFLAGS
10758 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10759 rc = VINF_SUCCESS;
10760 }
10761 break;
10762 }
10763
10764 default:
10765 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10766 rc = VINF_EM_RAW_EMULATE_INSTR;
10767 break;
10768 }
10769
10770 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10771 return rc;
10772}
10773
10774
10775/**
10776 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10777 * VM-exit.
10778 */
10779HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10780{
10781 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10782
10783 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10784 if (pVmxTransient->fWasGuestDebugStateActive)
10785 {
10786 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10787 HMVMX_RETURN_UNEXPECTED_EXIT();
10788 }
10789
10790 int rc = VERR_INTERNAL_ERROR_5;
10791 if ( !DBGFIsStepping(pVCpu)
10792 && !pVCpu->hm.s.fSingleInstruction
10793 && !pVmxTransient->fWasHyperDebugStateActive)
10794 {
10795 /* Don't intercept MOV DRx and #DB any more. */
10796 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10797 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10798 AssertRCReturn(rc, rc);
10799
10800 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10801 {
10802#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10803 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10804 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10805 AssertRCReturn(rc, rc);
10806#endif
10807 }
10808
10809 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10810 VMMRZCallRing3Disable(pVCpu);
10811 HM_DISABLE_PREEMPT_IF_NEEDED();
10812
10813 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10814 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10815 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10816
10817 HM_RESTORE_PREEMPT_IF_NEEDED();
10818 VMMRZCallRing3Enable(pVCpu);
10819
10820#ifdef VBOX_WITH_STATISTICS
10821 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10822 AssertRCReturn(rc, rc);
10823 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10824 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10825 else
10826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10827#endif
10828 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10829 return VINF_SUCCESS;
10830 }
10831
10832 /*
10833 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10834 * Update the segment registers and DR7 from the CPU.
10835 */
10836 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10837 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10838 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10839 AssertRCReturn(rc, rc);
10840 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10841
10842 PVM pVM = pVCpu->CTX_SUFF(pVM);
10843 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10844 {
10845 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10846 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10847 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10848 if (RT_SUCCESS(rc))
10849 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10850 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10851 }
10852 else
10853 {
10854 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10855 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10856 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10857 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10858 }
10859
10860 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10861 if (RT_SUCCESS(rc))
10862 {
10863 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10864 AssertRCReturn(rc2, rc2);
10865 }
10866 return rc;
10867}
10868
10869
10870/**
10871 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10872 * Conditional VM-exit.
10873 */
10874HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10875{
10876 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10877 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10878
10879 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10880 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10881 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10882 return VINF_SUCCESS;
10883 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10884 return rc;
10885
10886 RTGCPHYS GCPhys = 0;
10887 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10888
10889#if 0
10890 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10891#else
10892 /* Aggressive state sync. for now. */
10893 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10894 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10895 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10896#endif
10897 AssertRCReturn(rc, rc);
10898
10899 /*
10900 * If we succeed, resume guest execution.
10901 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10902 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10903 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10904 * weird case. See @bugref{6043}.
10905 */
10906 PVM pVM = pVCpu->CTX_SUFF(pVM);
10907 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10908 rc = VBOXSTRICTRC_VAL(rc2);
10909 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10910 if ( rc == VINF_SUCCESS
10911 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10912 || rc == VERR_PAGE_NOT_PRESENT)
10913 {
10914 /* Successfully handled MMIO operation. */
10915 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10916 | HM_CHANGED_GUEST_RSP
10917 | HM_CHANGED_GUEST_RFLAGS
10918 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10919 rc = VINF_SUCCESS;
10920 }
10921 return rc;
10922}
10923
10924
10925/**
10926 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10927 * VM-exit.
10928 */
10929HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10930{
10931 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10932 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10933
10934 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10935 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10936 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10937 return VINF_SUCCESS;
10938 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10939 return rc;
10940
10941 RTGCPHYS GCPhys = 0;
10942 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10943 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10944#if 0
10945 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10946#else
10947 /* Aggressive state sync. for now. */
10948 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10949 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10950 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10951#endif
10952 AssertRCReturn(rc, rc);
10953
10954 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10955 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10956
10957 RTGCUINT uErrorCode = 0;
10958 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10959 uErrorCode |= X86_TRAP_PF_ID;
10960 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10961 uErrorCode |= X86_TRAP_PF_RW;
10962 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10963 uErrorCode |= X86_TRAP_PF_P;
10964
10965 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10966
10967 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10968 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10969
10970 /* Handle the pagefault trap for the nested shadow table. */
10971 PVM pVM = pVCpu->CTX_SUFF(pVM);
10972 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10973 TRPMResetTrap(pVCpu);
10974
10975 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10976 if ( rc == VINF_SUCCESS
10977 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10978 || rc == VERR_PAGE_NOT_PRESENT)
10979 {
10980 /* Successfully synced our nested page tables. */
10981 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10982 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10983 | HM_CHANGED_GUEST_RSP
10984 | HM_CHANGED_GUEST_RFLAGS);
10985 return VINF_SUCCESS;
10986 }
10987
10988 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10989 return rc;
10990}
10991
10992/** @} */
10993
10994/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10995/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10996/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10997
10998/** @name VM-exit exception handlers.
10999 * @{
11000 */
11001
11002/**
11003 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11004 */
11005static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11006{
11007 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11009
11010 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11011 AssertRCReturn(rc, rc);
11012
11013 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11014 {
11015 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11016 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11017
11018 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11019 * provides VM-exit instruction length. If this causes problem later,
11020 * disassemble the instruction like it's done on AMD-V. */
11021 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11022 AssertRCReturn(rc2, rc2);
11023 return rc;
11024 }
11025
11026 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11027 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11028 return rc;
11029}
11030
11031
11032/**
11033 * VM-exit exception handler for #BP (Breakpoint exception).
11034 */
11035static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11036{
11037 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11039
11040 /** @todo Try optimize this by not saving the entire guest state unless
11041 * really needed. */
11042 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11043 AssertRCReturn(rc, rc);
11044
11045 PVM pVM = pVCpu->CTX_SUFF(pVM);
11046 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11047 if (rc == VINF_EM_RAW_GUEST_TRAP)
11048 {
11049 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11050 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11051 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11052 AssertRCReturn(rc, rc);
11053
11054 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11055 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11056 }
11057
11058 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11059 return rc;
11060}
11061
11062
11063/**
11064 * VM-exit exception handler for #DB (Debug exception).
11065 */
11066static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11067{
11068 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11069 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11070 Log6(("XcptDB\n"));
11071
11072 /*
11073 * Get the DR6-like values from the exit qualification and pass it to DBGF
11074 * for processing.
11075 */
11076 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11077 AssertRCReturn(rc, rc);
11078
11079 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11080 uint64_t uDR6 = X86_DR6_INIT_VAL;
11081 uDR6 |= ( pVmxTransient->uExitQualification
11082 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11083
11084 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11085 if (rc == VINF_EM_RAW_GUEST_TRAP)
11086 {
11087 /*
11088 * The exception was for the guest. Update DR6, DR7.GD and
11089 * IA32_DEBUGCTL.LBR before forwarding it.
11090 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11091 */
11092 VMMRZCallRing3Disable(pVCpu);
11093 HM_DISABLE_PREEMPT_IF_NEEDED();
11094
11095 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11096 pMixedCtx->dr[6] |= uDR6;
11097 if (CPUMIsGuestDebugStateActive(pVCpu))
11098 ASMSetDR6(pMixedCtx->dr[6]);
11099
11100 HM_RESTORE_PREEMPT_IF_NEEDED();
11101 VMMRZCallRing3Enable(pVCpu);
11102
11103 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11104 AssertRCReturn(rc, rc);
11105
11106 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11107 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11108
11109 /* Paranoia. */
11110 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11111 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11112
11113 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11114 AssertRCReturn(rc, rc);
11115
11116 /*
11117 * Raise #DB in the guest.
11118 *
11119 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11120 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11121 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11122 *
11123 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11124 */
11125 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11126 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11127 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11128 AssertRCReturn(rc, rc);
11129 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11130 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11131 return VINF_SUCCESS;
11132 }
11133
11134 /*
11135 * Not a guest trap, must be a hypervisor related debug event then.
11136 * Update DR6 in case someone is interested in it.
11137 */
11138 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11139 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11140 CPUMSetHyperDR6(pVCpu, uDR6);
11141
11142 return rc;
11143}
11144
11145
11146/**
11147 * VM-exit exception handler for #NM (Device-not-available exception: floating
11148 * point exception).
11149 */
11150static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11151{
11152 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11153
11154 /* We require CR0 and EFER. EFER is always up-to-date. */
11155 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11156 AssertRCReturn(rc, rc);
11157
11158 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11159 VMMRZCallRing3Disable(pVCpu);
11160 HM_DISABLE_PREEMPT_IF_NEEDED();
11161
11162 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11163 if (pVmxTransient->fWasGuestFPUStateActive)
11164 {
11165 rc = VINF_EM_RAW_GUEST_TRAP;
11166 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11167 }
11168 else
11169 {
11170#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11171 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11172#endif
11173 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11174 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11175 }
11176
11177 HM_RESTORE_PREEMPT_IF_NEEDED();
11178 VMMRZCallRing3Enable(pVCpu);
11179
11180 if (rc == VINF_SUCCESS)
11181 {
11182 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11183 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11184 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11185 pVCpu->hm.s.fUseGuestFpu = true;
11186 }
11187 else
11188 {
11189 /* Forward #NM to the guest. */
11190 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11191 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11192 AssertRCReturn(rc, rc);
11193 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11194 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11195 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11196 }
11197
11198 return VINF_SUCCESS;
11199}
11200
11201
11202/**
11203 * VM-exit exception handler for #GP (General-protection exception).
11204 *
11205 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11206 */
11207static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11208{
11209 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11210 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11211
11212 int rc = VERR_INTERNAL_ERROR_5;
11213 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11214 {
11215#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11216 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11217 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11218 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11219 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11220 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11221 AssertRCReturn(rc, rc);
11222 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11223 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11224 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11225 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11226 return rc;
11227#else
11228 /* We don't intercept #GP. */
11229 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11230 NOREF(pVmxTransient);
11231 return VERR_VMX_UNEXPECTED_EXCEPTION;
11232#endif
11233 }
11234
11235 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11236 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11237
11238 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11239 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11240 AssertRCReturn(rc, rc);
11241
11242 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11243 uint32_t cbOp = 0;
11244 PVM pVM = pVCpu->CTX_SUFF(pVM);
11245 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11246 if (RT_SUCCESS(rc))
11247 {
11248 rc = VINF_SUCCESS;
11249 Assert(cbOp == pDis->cbInstr);
11250 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11251 switch (pDis->pCurInstr->uOpcode)
11252 {
11253 case OP_CLI:
11254 {
11255 pMixedCtx->eflags.Bits.u1IF = 0;
11256 pMixedCtx->rip += pDis->cbInstr;
11257 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11258 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11260 break;
11261 }
11262
11263 case OP_STI:
11264 {
11265 pMixedCtx->eflags.Bits.u1IF = 1;
11266 pMixedCtx->rip += pDis->cbInstr;
11267 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11268 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11269 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11270 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11271 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11272 break;
11273 }
11274
11275 case OP_HLT:
11276 {
11277 rc = VINF_EM_HALT;
11278 pMixedCtx->rip += pDis->cbInstr;
11279 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11281 break;
11282 }
11283
11284 case OP_POPF:
11285 {
11286 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11287 uint32_t cbParm;
11288 uint32_t uMask;
11289 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11290 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11291 {
11292 cbParm = 4;
11293 uMask = 0xffffffff;
11294 }
11295 else
11296 {
11297 cbParm = 2;
11298 uMask = 0xffff;
11299 }
11300
11301 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11302 RTGCPTR GCPtrStack = 0;
11303 X86EFLAGS Eflags;
11304 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11305 &GCPtrStack);
11306 if (RT_SUCCESS(rc))
11307 {
11308 Assert(sizeof(Eflags.u32) >= cbParm);
11309 Eflags.u32 = 0;
11310 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11311 }
11312 if (RT_FAILURE(rc))
11313 {
11314 rc = VERR_EM_INTERPRETER;
11315 break;
11316 }
11317 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11318 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11319 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11320 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11321 pMixedCtx->esp += cbParm;
11322 pMixedCtx->esp &= uMask;
11323 pMixedCtx->rip += pDis->cbInstr;
11324 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11325 | HM_CHANGED_GUEST_RSP
11326 | HM_CHANGED_GUEST_RFLAGS);
11327 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11328 if (fStepping)
11329 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11330
11331 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11332 break;
11333 }
11334
11335 case OP_PUSHF:
11336 {
11337 uint32_t cbParm;
11338 uint32_t uMask;
11339 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11340 {
11341 cbParm = 4;
11342 uMask = 0xffffffff;
11343 }
11344 else
11345 {
11346 cbParm = 2;
11347 uMask = 0xffff;
11348 }
11349
11350 /* Get the stack pointer & push the contents of eflags onto the stack. */
11351 RTGCPTR GCPtrStack = 0;
11352 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11353 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11354 if (RT_FAILURE(rc))
11355 {
11356 rc = VERR_EM_INTERPRETER;
11357 break;
11358 }
11359 X86EFLAGS Eflags = pMixedCtx->eflags;
11360 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11361 Eflags.Bits.u1RF = 0;
11362 Eflags.Bits.u1VM = 0;
11363
11364 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11365 if (RT_FAILURE(rc))
11366 {
11367 rc = VERR_EM_INTERPRETER;
11368 break;
11369 }
11370 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11371 pMixedCtx->esp -= cbParm;
11372 pMixedCtx->esp &= uMask;
11373 pMixedCtx->rip += pDis->cbInstr;
11374 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11375 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11377 break;
11378 }
11379
11380 case OP_IRET:
11381 {
11382 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11383 * instruction reference. */
11384 RTGCPTR GCPtrStack = 0;
11385 uint32_t uMask = 0xffff;
11386 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11387 uint16_t aIretFrame[3];
11388 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11389 {
11390 rc = VERR_EM_INTERPRETER;
11391 break;
11392 }
11393 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11394 &GCPtrStack);
11395 if (RT_SUCCESS(rc))
11396 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11397 if (RT_FAILURE(rc))
11398 {
11399 rc = VERR_EM_INTERPRETER;
11400 break;
11401 }
11402 pMixedCtx->eip = 0;
11403 pMixedCtx->ip = aIretFrame[0];
11404 pMixedCtx->cs.Sel = aIretFrame[1];
11405 pMixedCtx->cs.ValidSel = aIretFrame[1];
11406 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11407 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11408 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11409 pMixedCtx->sp += sizeof(aIretFrame);
11410 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11411 | HM_CHANGED_GUEST_SEGMENT_REGS
11412 | HM_CHANGED_GUEST_RSP
11413 | HM_CHANGED_GUEST_RFLAGS);
11414 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11415 if (fStepping)
11416 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11417 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11419 break;
11420 }
11421
11422 case OP_INT:
11423 {
11424 uint16_t uVector = pDis->Param1.uValue & 0xff;
11425 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11426 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11427 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11428 break;
11429 }
11430
11431 case OP_INTO:
11432 {
11433 if (pMixedCtx->eflags.Bits.u1OF)
11434 {
11435 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11436 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11437 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11438 }
11439 break;
11440 }
11441
11442 default:
11443 {
11444 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11445 EMCODETYPE_SUPERVISOR);
11446 rc = VBOXSTRICTRC_VAL(rc2);
11447 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11448 /** @todo We have to set pending-debug exceptions here when the guest is
11449 * single-stepping depending on the instruction that was interpreted. */
11450 Log4(("#GP rc=%Rrc\n", rc));
11451 break;
11452 }
11453 }
11454 }
11455 else
11456 rc = VERR_EM_INTERPRETER;
11457
11458 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11459 ("#GP Unexpected rc=%Rrc\n", rc));
11460 return rc;
11461}
11462
11463
11464#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11465/**
11466 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11467 * the exception reported in the VMX transient structure back into the VM.
11468 *
11469 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11470 * up-to-date.
11471 */
11472static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11473{
11474 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11475
11476 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11477 hmR0VmxCheckExitDueToEventDelivery(). */
11478 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11479 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11480 AssertRCReturn(rc, rc);
11481 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11482
11483#ifdef DEBUG_ramshankar
11484 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11485 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11486 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11487#endif
11488
11489 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11490 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11491 return VINF_SUCCESS;
11492}
11493#endif
11494
11495
11496/**
11497 * VM-exit exception handler for #PF (Page-fault exception).
11498 */
11499static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11500{
11501 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11502 PVM pVM = pVCpu->CTX_SUFF(pVM);
11503 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11504 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11505 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11506 AssertRCReturn(rc, rc);
11507
11508#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11509 if (pVM->hm.s.fNestedPaging)
11510 {
11511 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11512 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11513 {
11514 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11515 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11516 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11517 }
11518 else
11519 {
11520 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11521 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11522 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11523 }
11524 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11525 return rc;
11526 }
11527#else
11528 Assert(!pVM->hm.s.fNestedPaging);
11529 NOREF(pVM);
11530#endif
11531
11532 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11533 AssertRCReturn(rc, rc);
11534
11535 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11536 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11537
11538 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11539 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11540 (RTGCPTR)pVmxTransient->uExitQualification);
11541
11542 Log4(("#PF: rc=%Rrc\n", rc));
11543 if (rc == VINF_SUCCESS)
11544 {
11545 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11546 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11547 * memory? We don't update the whole state here... */
11548 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11549 | HM_CHANGED_GUEST_RSP
11550 | HM_CHANGED_GUEST_RFLAGS
11551 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11552 TRPMResetTrap(pVCpu);
11553 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11554 return rc;
11555 }
11556 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11557 {
11558 if (!pVmxTransient->fVectoringPF)
11559 {
11560 /* It's a guest page fault and needs to be reflected to the guest. */
11561 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11562 TRPMResetTrap(pVCpu);
11563 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11564 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11565 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11566 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11567 }
11568 else
11569 {
11570 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11571 TRPMResetTrap(pVCpu);
11572 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11573 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11574 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11575 }
11576
11577 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11578 return VINF_SUCCESS;
11579 }
11580
11581 TRPMResetTrap(pVCpu);
11582 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11583 return rc;
11584}
11585
11586/** @} */
11587
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette