VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 96990

Last change on this file since 96990 was 96990, checked in by vboxsync, 2 years ago

VMM/HMVMXR0: Nested VMX: bugref:10092 Unintended newline.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 296.2 KB
Line 
1/* $Id: HMVMXR0.cpp 96990 2022-10-05 06:17:29Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2022 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_HM
33#define VMCPU_INCL_CPUM_GST_CTX
34#include <iprt/x86.h>
35#include <iprt/asm-amd64-x86.h>
36#include <iprt/thread.h>
37#include <iprt/mem.h>
38#include <iprt/mp.h>
39
40#include <VBox/vmm/pdmapi.h>
41#include <VBox/vmm/dbgf.h>
42#include <VBox/vmm/iem.h>
43#include <VBox/vmm/iom.h>
44#include <VBox/vmm/tm.h>
45#include <VBox/vmm/em.h>
46#include <VBox/vmm/gcm.h>
47#include <VBox/vmm/gim.h>
48#include <VBox/vmm/apic.h>
49#include "HMInternal.h"
50#include <VBox/vmm/vmcc.h>
51#include <VBox/vmm/hmvmxinline.h>
52#include "HMVMXR0.h"
53#include "VMXInternal.h"
54#include "dtrace/VBoxVMM.h"
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60#ifdef DEBUG_ramshankar
61# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
62# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
63# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
64# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
65# define HMVMX_ALWAYS_CLEAN_TRANSIENT
66# define HMVMX_ALWAYS_CHECK_GUEST_STATE
67# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
68# define HMVMX_ALWAYS_TRAP_PF
69# define HMVMX_ALWAYS_FLUSH_TLB
70# define HMVMX_ALWAYS_SWAP_EFER
71#endif
72
73
74/*********************************************************************************************************************************
75* Structures and Typedefs *
76*********************************************************************************************************************************/
77/**
78 * VMX page allocation information.
79 */
80typedef struct
81{
82 uint32_t fValid; /**< Whether to allocate this page (e.g, based on a CPU feature). */
83 uint32_t uPadding0; /**< Padding to ensure array of these structs are aligned to a multiple of 8. */
84 PRTHCPHYS pHCPhys; /**< Where to store the host-physical address of the allocation. */
85 PRTR0PTR ppVirt; /**< Where to store the host-virtual address of the allocation. */
86} VMXPAGEALLOCINFO;
87/** Pointer to VMX page-allocation info. */
88typedef VMXPAGEALLOCINFO *PVMXPAGEALLOCINFO;
89/** Pointer to a const VMX page-allocation info. */
90typedef const VMXPAGEALLOCINFO *PCVMXPAGEALLOCINFO;
91AssertCompileSizeAlignment(VMXPAGEALLOCINFO, 8);
92
93
94/*********************************************************************************************************************************
95* Internal Functions *
96*********************************************************************************************************************************/
97static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient);
98static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo);
99
100
101/**
102 * Checks if the given MSR is part of the lastbranch-from-IP MSR stack.
103 * @returns @c true if it's part of LBR stack, @c false otherwise.
104 *
105 * @param pVM The cross context VM structure.
106 * @param idMsr The MSR.
107 * @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
108 * Optional, can be NULL.
109 *
110 * @remarks Must only be called when LBR is enabled.
111 */
112DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchFromMsr(PCVMCC pVM, uint32_t idMsr, uint32_t *pidxMsr)
113{
114 Assert(pVM->hmr0.s.vmx.fLbr);
115 Assert(pVM->hmr0.s.vmx.idLbrFromIpMsrFirst);
116 uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrFromIpMsrLast - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst + 1;
117 uint32_t const idxMsr = idMsr - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst;
118 if (idxMsr < cLbrStack)
119 {
120 if (pidxMsr)
121 *pidxMsr = idxMsr;
122 return true;
123 }
124 return false;
125}
126
127
128/**
129 * Checks if the given MSR is part of the lastbranch-to-IP MSR stack.
130 * @returns @c true if it's part of LBR stack, @c false otherwise.
131 *
132 * @param pVM The cross context VM structure.
133 * @param idMsr The MSR.
134 * @param pidxMsr Where to store the index of the MSR in the LBR MSR array.
135 * Optional, can be NULL.
136 *
137 * @remarks Must only be called when LBR is enabled and when lastbranch-to-IP MSRs
138 * are supported by the CPU (see hmR0VmxSetupLbrMsrRange).
139 */
140DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchToMsr(PCVMCC pVM, uint32_t idMsr, uint32_t *pidxMsr)
141{
142 Assert(pVM->hmr0.s.vmx.fLbr);
143 if (pVM->hmr0.s.vmx.idLbrToIpMsrFirst)
144 {
145 uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrToIpMsrLast - pVM->hmr0.s.vmx.idLbrToIpMsrFirst + 1;
146 uint32_t const idxMsr = idMsr - pVM->hmr0.s.vmx.idLbrToIpMsrFirst;
147 if (idxMsr < cLbrStack)
148 {
149 if (pidxMsr)
150 *pidxMsr = idxMsr;
151 return true;
152 }
153 }
154 return false;
155}
156
157
158/**
159 * Gets the active (in use) VMCS info. object for the specified VCPU.
160 *
161 * This is either the guest or nested-guest VMCS info. and need not necessarily
162 * pertain to the "current" VMCS (in the VMX definition of the term). For instance,
163 * if the VM-entry failed due to an invalid-guest state, we may have "cleared" the
164 * current VMCS while returning to ring-3. However, the VMCS info. object for that
165 * VMCS would still be active and returned here so that we could dump the VMCS
166 * fields to ring-3 for diagnostics. This function is thus only used to
167 * distinguish between the nested-guest or guest VMCS.
168 *
169 * @returns The active VMCS information.
170 * @param pVCpu The cross context virtual CPU structure.
171 *
172 * @thread EMT.
173 * @remarks This function may be called with preemption or interrupts disabled!
174 */
175DECLINLINE(PVMXVMCSINFO) hmGetVmxActiveVmcsInfo(PVMCPUCC pVCpu)
176{
177 if (!pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
178 return &pVCpu->hmr0.s.vmx.VmcsInfo;
179 return &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
180}
181
182
183/**
184 * Returns whether the VM-exit MSR-store area differs from the VM-exit MSR-load
185 * area.
186 *
187 * @returns @c true if it's different, @c false otherwise.
188 * @param pVmcsInfo The VMCS info. object.
189 */
190DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
191{
192 return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
193 && pVmcsInfo->pvGuestMsrStore);
194}
195
196
197/**
198 * Sets the given Processor-based VM-execution controls.
199 *
200 * @param pVmxTransient The VMX-transient structure.
201 * @param uProcCtls The Processor-based VM-execution controls to set.
202 */
203static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
204{
205 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
206 if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
207 {
208 pVmcsInfo->u32ProcCtls |= uProcCtls;
209 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
210 AssertRC(rc);
211 }
212}
213
214
215/**
216 * Removes the given Processor-based VM-execution controls.
217 *
218 * @param pVCpu The cross context virtual CPU structure.
219 * @param pVmxTransient The VMX-transient structure.
220 * @param uProcCtls The Processor-based VM-execution controls to remove.
221 *
222 * @remarks When executing a nested-guest, this will not remove any of the specified
223 * controls if the nested hypervisor has set any one of them.
224 */
225static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
226{
227 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
228 if (pVmcsInfo->u32ProcCtls & uProcCtls)
229 {
230#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
231 if ( !pVmxTransient->fIsNestedGuest
232 || !CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uProcCtls))
233#else
234 NOREF(pVCpu);
235 if (!pVmxTransient->fIsNestedGuest)
236#endif
237 {
238 pVmcsInfo->u32ProcCtls &= ~uProcCtls;
239 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
240 AssertRC(rc);
241 }
242 }
243}
244
245
246/**
247 * Sets the TSC offset for the current VMCS.
248 *
249 * @param uTscOffset The TSC offset to set.
250 * @param pVmcsInfo The VMCS info. object.
251 */
252static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
253{
254 if (pVmcsInfo->u64TscOffset != uTscOffset)
255 {
256 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
257 AssertRC(rc);
258 pVmcsInfo->u64TscOffset = uTscOffset;
259 }
260}
261
262
263/**
264 * Loads the VMCS specified by the VMCS info. object.
265 *
266 * @returns VBox status code.
267 * @param pVmcsInfo The VMCS info. object.
268 *
269 * @remarks Can be called with interrupts disabled.
270 */
271static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
272{
273 Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
274 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
275
276 int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
277 if (RT_SUCCESS(rc))
278 pVmcsInfo->fVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
279 return rc;
280}
281
282
283/**
284 * Clears the VMCS specified by the VMCS info. object.
285 *
286 * @returns VBox status code.
287 * @param pVmcsInfo The VMCS info. object.
288 *
289 * @remarks Can be called with interrupts disabled.
290 */
291static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
292{
293 Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
294 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
295
296 int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
297 if (RT_SUCCESS(rc))
298 pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
299 return rc;
300}
301
302
303/**
304 * Checks whether the MSR belongs to the set of guest MSRs that we restore
305 * lazily while leaving VT-x.
306 *
307 * @returns true if it does, false otherwise.
308 * @param pVCpu The cross context virtual CPU structure.
309 * @param idMsr The MSR to check.
310 */
311static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
312{
313 if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fAllow64BitGuests)
314 {
315 switch (idMsr)
316 {
317 case MSR_K8_LSTAR:
318 case MSR_K6_STAR:
319 case MSR_K8_SF_MASK:
320 case MSR_K8_KERNEL_GS_BASE:
321 return true;
322 }
323 }
324 return false;
325}
326
327
328/**
329 * Loads a set of guests MSRs to allow read/passthru to the guest.
330 *
331 * The name of this function is slightly confusing. This function does NOT
332 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
333 * common prefix for functions dealing with "lazy restoration" of the shared
334 * MSRs.
335 *
336 * @param pVCpu The cross context virtual CPU structure.
337 *
338 * @remarks No-long-jump zone!!!
339 */
340static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
341{
342 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
343 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
344
345 Assert(pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
346 if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fAllow64BitGuests)
347 {
348 /*
349 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
350 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
351 * we can skip a few MSR writes.
352 *
353 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
354 * guest MSR values in the guest-CPU context might be different to what's currently
355 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
356 * CPU, see @bugref{8728}.
357 */
358 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
359 if ( !(pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
360 && pCtx->msrKERNELGSBASE == pVCpu->hmr0.s.vmx.u64HostMsrKernelGsBase
361 && pCtx->msrLSTAR == pVCpu->hmr0.s.vmx.u64HostMsrLStar
362 && pCtx->msrSTAR == pVCpu->hmr0.s.vmx.u64HostMsrStar
363 && pCtx->msrSFMASK == pVCpu->hmr0.s.vmx.u64HostMsrSfMask)
364 {
365#ifdef VBOX_STRICT
366 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
367 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
368 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
369 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
370#endif
371 }
372 else
373 {
374 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
375 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
376 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
377 /* The system call flag mask register isn't as benign and accepting of all
378 values as the above, so mask it to avoid #GP'ing on corrupted input. */
379 Assert(!(pCtx->msrSFMASK & ~(uint64_t)UINT32_MAX));
380 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK & UINT32_MAX);
381 }
382 }
383 pVCpu->hmr0.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
384}
385
386
387/**
388 * Checks if the specified guest MSR is part of the VM-entry MSR-load area.
389 *
390 * @returns @c true if found, @c false otherwise.
391 * @param pVmcsInfo The VMCS info. object.
392 * @param idMsr The MSR to find.
393 */
394static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
395{
396 PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
397 uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
398 Assert(pMsrs);
399 Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
400 for (uint32_t i = 0; i < cMsrs; i++)
401 {
402 if (pMsrs[i].u32Msr == idMsr)
403 return true;
404 }
405 return false;
406}
407
408
409/**
410 * Performs lazy restoration of the set of host MSRs if they were previously
411 * loaded with guest MSR values.
412 *
413 * @param pVCpu The cross context virtual CPU structure.
414 *
415 * @remarks No-long-jump zone!!!
416 * @remarks The guest MSRs should have been saved back into the guest-CPU
417 * context by hmR0VmxImportGuestState()!!!
418 */
419static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
420{
421 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
422 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
423
424 if (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
425 {
426 Assert(pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
427 if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fAllow64BitGuests)
428 {
429 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hmr0.s.vmx.u64HostMsrLStar);
430 ASMWrMsr(MSR_K6_STAR, pVCpu->hmr0.s.vmx.u64HostMsrStar);
431 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hmr0.s.vmx.u64HostMsrSfMask);
432 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hmr0.s.vmx.u64HostMsrKernelGsBase);
433 }
434 }
435 pVCpu->hmr0.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
436}
437
438
439/**
440 * Sets pfnStartVm to the best suited variant.
441 *
442 * This must be called whenever anything changes relative to the hmR0VmXStartVm
443 * variant selection:
444 * - pVCpu->hm.s.fLoadSaveGuestXcr0
445 * - HM_WSF_IBPB_ENTRY in pVCpu->hmr0.s.fWorldSwitcher
446 * - HM_WSF_IBPB_EXIT in pVCpu->hmr0.s.fWorldSwitcher
447 * - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
448 * - Perhaps: CPUMCTX.fXStateMask (windows only)
449 *
450 * We currently ASSUME that neither HM_WSF_IBPB_ENTRY nor HM_WSF_IBPB_EXIT
451 * cannot be changed at runtime.
452 */
453static void hmR0VmxUpdateStartVmFunction(PVMCPUCC pVCpu)
454{
455 static const struct CLANGWORKAROUND { PFNHMVMXSTARTVM pfn; } s_aHmR0VmxStartVmFunctions[] =
456 {
457 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit },
458 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit },
459 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit },
460 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit },
461 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit },
462 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit },
463 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit },
464 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit },
465 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit },
466 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit },
467 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit },
468 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit },
469 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit },
470 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit },
471 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit },
472 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit },
473 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit },
474 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit },
475 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit },
476 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit },
477 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit },
478 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit },
479 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit },
480 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit },
481 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit },
482 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit },
483 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit },
484 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit },
485 { hmR0VmxStartVm_SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit },
486 { hmR0VmxStartVm_WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit },
487 { hmR0VmxStartVm_SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit },
488 { hmR0VmxStartVm_WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit },
489 };
490 uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
491 | (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
492 | (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_L1D_ENTRY ? 4 : 0)
493 | (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_MDS_ENTRY ? 8 : 0)
494 | (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 16 : 0);
495 PFNHMVMXSTARTVM const pfnStartVm = s_aHmR0VmxStartVmFunctions[idx].pfn;
496 if (pVCpu->hmr0.s.vmx.pfnStartVm != pfnStartVm)
497 pVCpu->hmr0.s.vmx.pfnStartVm = pfnStartVm;
498}
499
500
501/**
502 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
503 * stack.
504 *
505 * @returns Strict VBox status code (i.e. informational status codes too).
506 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
507 * @param pVCpu The cross context virtual CPU structure.
508 * @param uValue The value to push to the guest stack.
509 */
510static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
511{
512 /*
513 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
514 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
515 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
516 */
517 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
518 if (pCtx->sp == 1)
519 return VINF_EM_RESET;
520 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
521 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
522 AssertRC(rc);
523 return rc;
524}
525
526
527/**
528 * Wrapper around VMXWriteVmcs16 taking a pVCpu parameter so VCC doesn't complain about
529 * unreferenced local parameters in the template code...
530 */
531DECL_FORCE_INLINE(int) hmR0VmxWriteVmcs16(PCVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t u16Val)
532{
533 RT_NOREF(pVCpu);
534 return VMXWriteVmcs16(uFieldEnc, u16Val);
535}
536
537
538/**
539 * Wrapper around VMXWriteVmcs32 taking a pVCpu parameter so VCC doesn't complain about
540 * unreferenced local parameters in the template code...
541 */
542DECL_FORCE_INLINE(int) hmR0VmxWriteVmcs32(PCVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t u32Val)
543{
544 RT_NOREF(pVCpu);
545 return VMXWriteVmcs32(uFieldEnc, u32Val);
546}
547
548
549/**
550 * Wrapper around VMXWriteVmcs64 taking a pVCpu parameter so VCC doesn't complain about
551 * unreferenced local parameters in the template code...
552 */
553DECL_FORCE_INLINE(int) hmR0VmxWriteVmcs64(PCVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t u64Val)
554{
555 RT_NOREF(pVCpu);
556 return VMXWriteVmcs64(uFieldEnc, u64Val);
557}
558
559
560/**
561 * Wrapper around VMXReadVmcs16 taking a pVCpu parameter so VCC doesn't complain about
562 * unreferenced local parameters in the template code...
563 */
564DECL_FORCE_INLINE(int) hmR0VmxReadVmcs16(PCVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t *pu16Val)
565{
566 RT_NOREF(pVCpu);
567 return VMXReadVmcs16(uFieldEnc, pu16Val);
568}
569
570
571/**
572 * Wrapper around VMXReadVmcs32 taking a pVCpu parameter so VCC doesn't complain about
573 * unreferenced local parameters in the template code...
574 */
575DECL_FORCE_INLINE(int) hmR0VmxReadVmcs32(PCVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t *pu32Val)
576{
577 RT_NOREF(pVCpu);
578 return VMXReadVmcs32(uFieldEnc, pu32Val);
579}
580
581
582/**
583 * Wrapper around VMXReadVmcs64 taking a pVCpu parameter so VCC doesn't complain about
584 * unreferenced local parameters in the template code...
585 */
586DECL_FORCE_INLINE(int) hmR0VmxReadVmcs64(PCVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t *pu64Val)
587{
588 RT_NOREF(pVCpu);
589 return VMXReadVmcs64(uFieldEnc, pu64Val);
590}
591
592
593/*
594 * Instantiate the code we share with the NEM darwin backend.
595 */
596#define VCPU_2_VMXSTATE(a_pVCpu) (a_pVCpu)->hm.s
597#define VCPU_2_VMXSTATS(a_pVCpu) (a_pVCpu)->hm.s
598
599#define VM_IS_VMX_UNRESTRICTED_GUEST(a_pVM) (a_pVM)->hmr0.s.vmx.fUnrestrictedGuest
600#define VM_IS_VMX_NESTED_PAGING(a_pVM) (a_pVM)->hmr0.s.fNestedPaging
601#define VM_IS_VMX_PREEMPT_TIMER_USED(a_pVM) (a_pVM)->hmr0.s.vmx.fUsePreemptTimer
602#define VM_IS_VMX_LBR(a_pVM) (a_pVM)->hmr0.s.vmx.fLbr
603
604#define VMX_VMCS_WRITE_16(a_pVCpu, a_FieldEnc, a_Val) hmR0VmxWriteVmcs16((a_pVCpu), (a_FieldEnc), (a_Val))
605#define VMX_VMCS_WRITE_32(a_pVCpu, a_FieldEnc, a_Val) hmR0VmxWriteVmcs32((a_pVCpu), (a_FieldEnc), (a_Val))
606#define VMX_VMCS_WRITE_64(a_pVCpu, a_FieldEnc, a_Val) hmR0VmxWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
607#define VMX_VMCS_WRITE_NW(a_pVCpu, a_FieldEnc, a_Val) hmR0VmxWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
608
609#define VMX_VMCS_READ_16(a_pVCpu, a_FieldEnc, a_pVal) hmR0VmxReadVmcs16((a_pVCpu), (a_FieldEnc), (a_pVal))
610#define VMX_VMCS_READ_32(a_pVCpu, a_FieldEnc, a_pVal) hmR0VmxReadVmcs32((a_pVCpu), (a_FieldEnc), (a_pVal))
611#define VMX_VMCS_READ_64(a_pVCpu, a_FieldEnc, a_pVal) hmR0VmxReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
612#define VMX_VMCS_READ_NW(a_pVCpu, a_FieldEnc, a_pVal) hmR0VmxReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
613
614#include "../VMMAll/VMXAllTemplate.cpp.h"
615
616#undef VMX_VMCS_WRITE_16
617#undef VMX_VMCS_WRITE_32
618#undef VMX_VMCS_WRITE_64
619#undef VMX_VMCS_WRITE_NW
620
621#undef VMX_VMCS_READ_16
622#undef VMX_VMCS_READ_32
623#undef VMX_VMCS_READ_64
624#undef VMX_VMCS_READ_NW
625
626#undef VM_IS_VMX_PREEMPT_TIMER_USED
627#undef VM_IS_VMX_NESTED_PAGING
628#undef VM_IS_VMX_UNRESTRICTED_GUEST
629#undef VCPU_2_VMXSTATS
630#undef VCPU_2_VMXSTATE
631
632
633/**
634 * Updates the VM's last error record.
635 *
636 * If there was a VMX instruction error, reads the error data from the VMCS and
637 * updates VCPU's last error record as well.
638 *
639 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
640 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
641 * VERR_VMX_INVALID_VMCS_FIELD.
642 * @param rc The error code.
643 */
644static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
645{
646 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
647 || rc == VERR_VMX_UNABLE_TO_START_VM)
648 {
649 AssertPtrReturnVoid(pVCpu);
650 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
651 }
652 pVCpu->CTX_SUFF(pVM)->hm.s.ForR3.rcInit = rc;
653}
654
655
656/**
657 * Enters VMX root mode operation on the current CPU.
658 *
659 * @returns VBox status code.
660 * @param pHostCpu The HM physical-CPU structure.
661 * @param pVM The cross context VM structure. Can be
662 * NULL, after a resume.
663 * @param HCPhysCpuPage Physical address of the VMXON region.
664 * @param pvCpuPage Pointer to the VMXON region.
665 */
666static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
667{
668 Assert(pHostCpu);
669 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
670 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
671 Assert(pvCpuPage);
672 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
673
674 if (pVM)
675 {
676 /* Write the VMCS revision identifier to the VMXON region. */
677 *(uint32_t *)pvCpuPage = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID);
678 }
679
680 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
681 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
682
683 /* Enable the VMX bit in CR4 if necessary. */
684 RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
685
686 /* Record whether VMXE was already prior to us enabling it above. */
687 pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
688
689 /* Enter VMX root mode. */
690 int rc = VMXEnable(HCPhysCpuPage);
691 if (RT_FAILURE(rc))
692 {
693 /* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
694 if (!pHostCpu->fVmxeAlreadyEnabled)
695 SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
696
697 if (pVM)
698 pVM->hm.s.ForR3.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
699 }
700
701 /* Restore interrupts. */
702 ASMSetFlags(fEFlags);
703 return rc;
704}
705
706
707/**
708 * Exits VMX root mode operation on the current CPU.
709 *
710 * @returns VBox status code.
711 * @param pHostCpu The HM physical-CPU structure.
712 */
713static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
714{
715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
716
717 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
718 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
719
720 /* If we're for some reason not in VMX root mode, then don't leave it. */
721 RTCCUINTREG const uHostCr4 = ASMGetCR4();
722
723 int rc;
724 if (uHostCr4 & X86_CR4_VMXE)
725 {
726 /* Exit VMX root mode and clear the VMX bit in CR4. */
727 VMXDisable();
728
729 /* Clear CR4.VMXE only if it was clear prior to use setting it. */
730 if (!pHostCpu->fVmxeAlreadyEnabled)
731 SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
732
733 rc = VINF_SUCCESS;
734 }
735 else
736 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
737
738 /* Restore interrupts. */
739 ASMSetFlags(fEFlags);
740 return rc;
741}
742
743
744/**
745 * Allocates pages specified as specified by an array of VMX page allocation info
746 * objects.
747 *
748 * The pages contents are zero'd after allocation.
749 *
750 * @returns VBox status code.
751 * @param phMemObj Where to return the handle to the allocation.
752 * @param paAllocInfo The pointer to the first element of the VMX
753 * page-allocation info object array.
754 * @param cEntries The number of elements in the @a paAllocInfo array.
755 */
756static int hmR0VmxPagesAllocZ(PRTR0MEMOBJ phMemObj, PVMXPAGEALLOCINFO paAllocInfo, uint32_t cEntries)
757{
758 *phMemObj = NIL_RTR0MEMOBJ;
759
760 /* Figure out how many pages to allocate. */
761 uint32_t cPages = 0;
762 for (uint32_t iPage = 0; iPage < cEntries; iPage++)
763 cPages += !!paAllocInfo[iPage].fValid;
764
765 /* Allocate the pages. */
766 if (cPages)
767 {
768 size_t const cbPages = cPages << HOST_PAGE_SHIFT;
769 int rc = RTR0MemObjAllocPage(phMemObj, cbPages, false /* fExecutable */);
770 if (RT_FAILURE(rc))
771 return rc;
772
773 /* Zero the contents and assign each page to the corresponding VMX page-allocation entry. */
774 void *pvFirstPage = RTR0MemObjAddress(*phMemObj);
775 RT_BZERO(pvFirstPage, cbPages);
776
777 uint32_t iPage = 0;
778 for (uint32_t i = 0; i < cEntries; i++)
779 if (paAllocInfo[i].fValid)
780 {
781 RTHCPHYS const HCPhysPage = RTR0MemObjGetPagePhysAddr(*phMemObj, iPage);
782 void *pvPage = (void *)((uintptr_t)pvFirstPage + (iPage << X86_PAGE_4K_SHIFT));
783 Assert(HCPhysPage && HCPhysPage != NIL_RTHCPHYS);
784 AssertPtr(pvPage);
785
786 Assert(paAllocInfo[iPage].pHCPhys);
787 Assert(paAllocInfo[iPage].ppVirt);
788 *paAllocInfo[iPage].pHCPhys = HCPhysPage;
789 *paAllocInfo[iPage].ppVirt = pvPage;
790
791 /* Move to next page. */
792 ++iPage;
793 }
794
795 /* Make sure all valid (requested) pages have been assigned. */
796 Assert(iPage == cPages);
797 }
798 return VINF_SUCCESS;
799}
800
801
802/**
803 * Frees pages allocated using hmR0VmxPagesAllocZ.
804 *
805 * @param phMemObj Pointer to the memory object handle. Will be set to
806 * NIL.
807 */
808DECL_FORCE_INLINE(void) hmR0VmxPagesFree(PRTR0MEMOBJ phMemObj)
809{
810 /* We can cleanup wholesale since it's all one allocation. */
811 if (*phMemObj != NIL_RTR0MEMOBJ)
812 {
813 RTR0MemObjFree(*phMemObj, true /* fFreeMappings */);
814 *phMemObj = NIL_RTR0MEMOBJ;
815 }
816}
817
818
819/**
820 * Initializes a VMCS info. object.
821 *
822 * @param pVmcsInfo The VMCS info. object.
823 * @param pVmcsInfoShared The VMCS info. object shared with ring-3.
824 */
825static void hmR0VmxVmcsInfoInit(PVMXVMCSINFO pVmcsInfo, PVMXVMCSINFOSHARED pVmcsInfoShared)
826{
827 RT_ZERO(*pVmcsInfo);
828 RT_ZERO(*pVmcsInfoShared);
829
830 pVmcsInfo->pShared = pVmcsInfoShared;
831 Assert(pVmcsInfo->hMemObj == NIL_RTR0MEMOBJ);
832 pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
833 pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
834 pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
835 pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
836 pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
837 pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
838 pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
839 pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
840 pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
841 pVmcsInfo->idHostCpuState = NIL_RTCPUID;
842 pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
843}
844
845
846/**
847 * Frees the VT-x structures for a VMCS info. object.
848 *
849 * @param pVmcsInfo The VMCS info. object.
850 * @param pVmcsInfoShared The VMCS info. object shared with ring-3.
851 */
852static void hmR0VmxVmcsInfoFree(PVMXVMCSINFO pVmcsInfo, PVMXVMCSINFOSHARED pVmcsInfoShared)
853{
854 hmR0VmxPagesFree(&pVmcsInfo->hMemObj);
855 hmR0VmxVmcsInfoInit(pVmcsInfo, pVmcsInfoShared);
856}
857
858
859/**
860 * Allocates the VT-x structures for a VMCS info. object.
861 *
862 * @returns VBox status code.
863 * @param pVCpu The cross context virtual CPU structure.
864 * @param pVmcsInfo The VMCS info. object.
865 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
866 *
867 * @remarks The caller is expected to take care of any and all allocation failures.
868 * This function will not perform any cleanup for failures half-way
869 * through.
870 */
871static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
872{
873 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
874
875 bool const fMsrBitmaps = RT_BOOL(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS);
876 bool const fShadowVmcs = !fIsNstGstVmcs ? pVM->hmr0.s.vmx.fUseVmcsShadowing : pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing;
877 Assert(!pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing); /* VMCS shadowing is not yet exposed to the guest. */
878 VMXPAGEALLOCINFO aAllocInfo[] =
879 {
880 { true, 0 /* Unused */, &pVmcsInfo->HCPhysVmcs, &pVmcsInfo->pvVmcs },
881 { true, 0 /* Unused */, &pVmcsInfo->HCPhysGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad },
882 { true, 0 /* Unused */, &pVmcsInfo->HCPhysHostMsrLoad, &pVmcsInfo->pvHostMsrLoad },
883 { fMsrBitmaps, 0 /* Unused */, &pVmcsInfo->HCPhysMsrBitmap, &pVmcsInfo->pvMsrBitmap },
884 { fShadowVmcs, 0 /* Unused */, &pVmcsInfo->HCPhysShadowVmcs, &pVmcsInfo->pvShadowVmcs },
885 };
886
887 int rc = hmR0VmxPagesAllocZ(&pVmcsInfo->hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
888 if (RT_FAILURE(rc))
889 return rc;
890
891 /*
892 * We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
893 * Because they contain a symmetric list of guest MSRs to load on VM-entry and store on VM-exit.
894 */
895 AssertCompile(RT_ELEMENTS(aAllocInfo) > 0);
896 Assert(pVmcsInfo->HCPhysGuestMsrLoad != NIL_RTHCPHYS);
897 pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
898 pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
899
900 /*
901 * Get the virtual-APIC page rather than allocating them again.
902 */
903 if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
904 {
905 if (!fIsNstGstVmcs)
906 {
907 if (PDMHasApic(pVM))
908 {
909 rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /*pR3Ptr*/);
910 if (RT_FAILURE(rc))
911 return rc;
912 Assert(pVmcsInfo->pbVirtApic);
913 Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
914 }
915 }
916 else
917 {
918 pVmcsInfo->pbVirtApic = &pVCpu->cpum.GstCtx.hwvirt.vmx.abVirtApicPage[0];
919 pVmcsInfo->HCPhysVirtApic = GVMMR0ConvertGVMPtr2HCPhys(pVM, pVmcsInfo->pbVirtApic);
920 Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
921 }
922 }
923
924 return VINF_SUCCESS;
925}
926
927
928/**
929 * Free all VT-x structures for the VM.
930 *
931 * @returns IPRT status code.
932 * @param pVM The cross context VM structure.
933 */
934static void hmR0VmxStructsFree(PVMCC pVM)
935{
936 hmR0VmxPagesFree(&pVM->hmr0.s.vmx.hMemObj);
937#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
938 if (pVM->hmr0.s.vmx.fUseVmcsShadowing)
939 {
940 RTMemFree(pVM->hmr0.s.vmx.paShadowVmcsFields);
941 pVM->hmr0.s.vmx.paShadowVmcsFields = NULL;
942 RTMemFree(pVM->hmr0.s.vmx.paShadowVmcsRoFields);
943 pVM->hmr0.s.vmx.paShadowVmcsRoFields = NULL;
944 }
945#endif
946
947 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
948 {
949 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
950 hmR0VmxVmcsInfoFree(&pVCpu->hmr0.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfo);
951#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
952 if (pVM->cpum.ro.GuestFeatures.fVmx)
953 hmR0VmxVmcsInfoFree(&pVCpu->hmr0.s.vmx.VmcsInfoNstGst, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
954#endif
955 }
956}
957
958
959/**
960 * Allocate all VT-x structures for the VM.
961 *
962 * @returns IPRT status code.
963 * @param pVM The cross context VM structure.
964 *
965 * @remarks This functions will cleanup on memory allocation failures.
966 */
967static int hmR0VmxStructsAlloc(PVMCC pVM)
968{
969 /*
970 * Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
971 * The VMCS size cannot be more than 4096 bytes.
972 *
973 * See Intel spec. Appendix A.1 "Basic VMX Information".
974 */
975 uint32_t const cbVmcs = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
976 if (cbVmcs <= X86_PAGE_4K_SIZE)
977 { /* likely */ }
978 else
979 {
980 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
981 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
982 }
983
984 /*
985 * Allocate per-VM VT-x structures.
986 */
987 bool const fVirtApicAccess = RT_BOOL(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
988 bool const fUseVmcsShadowing = pVM->hmr0.s.vmx.fUseVmcsShadowing;
989 VMXPAGEALLOCINFO aAllocInfo[] =
990 {
991 { fVirtApicAccess, 0 /* Unused */, &pVM->hmr0.s.vmx.HCPhysApicAccess, (PRTR0PTR)&pVM->hmr0.s.vmx.pbApicAccess },
992 { fUseVmcsShadowing, 0 /* Unused */, &pVM->hmr0.s.vmx.HCPhysVmreadBitmap, &pVM->hmr0.s.vmx.pvVmreadBitmap },
993 { fUseVmcsShadowing, 0 /* Unused */, &pVM->hmr0.s.vmx.HCPhysVmwriteBitmap, &pVM->hmr0.s.vmx.pvVmwriteBitmap },
994#ifdef VBOX_WITH_CRASHDUMP_MAGIC
995 { true, 0 /* Unused */, &pVM->hmr0.s.vmx.HCPhysScratch, (PRTR0PTR)&pVM->hmr0.s.vmx.pbScratch },
996#endif
997 };
998
999 int rc = hmR0VmxPagesAllocZ(&pVM->hmr0.s.vmx.hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
1000 if (RT_SUCCESS(rc))
1001 {
1002#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1003 /* Allocate the shadow VMCS-fields array. */
1004 if (fUseVmcsShadowing)
1005 {
1006 Assert(!pVM->hmr0.s.vmx.cShadowVmcsFields);
1007 Assert(!pVM->hmr0.s.vmx.cShadowVmcsRoFields);
1008 pVM->hmr0.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1009 pVM->hmr0.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1010 if (!pVM->hmr0.s.vmx.paShadowVmcsFields || !pVM->hmr0.s.vmx.paShadowVmcsRoFields)
1011 rc = VERR_NO_MEMORY;
1012 }
1013#endif
1014
1015 /*
1016 * Allocate per-VCPU VT-x structures.
1017 */
1018 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus && RT_SUCCESS(rc); idCpu++)
1019 {
1020 /* Allocate the guest VMCS structures. */
1021 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1022 rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hmr0.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1023
1024#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1025 /* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1026 if (pVM->cpum.ro.GuestFeatures.fVmx && RT_SUCCESS(rc))
1027 rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hmr0.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1028#endif
1029 }
1030 if (RT_SUCCESS(rc))
1031 return VINF_SUCCESS;
1032 }
1033 hmR0VmxStructsFree(pVM);
1034 return rc;
1035}
1036
1037
1038/**
1039 * Pre-initializes non-zero fields in VMX structures that will be allocated.
1040 *
1041 * @param pVM The cross context VM structure.
1042 */
1043static void hmR0VmxStructsInit(PVMCC pVM)
1044{
1045 /* Paranoia. */
1046 Assert(pVM->hmr0.s.vmx.pbApicAccess == NULL);
1047#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1048 Assert(pVM->hmr0.s.vmx.pbScratch == NULL);
1049#endif
1050
1051 /*
1052 * Initialize members up-front so we can cleanup en masse on allocation failures.
1053 */
1054#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1055 pVM->hmr0.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1056#endif
1057 pVM->hmr0.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1058 pVM->hmr0.s.vmx.HCPhysVmreadBitmap = NIL_RTHCPHYS;
1059 pVM->hmr0.s.vmx.HCPhysVmwriteBitmap = NIL_RTHCPHYS;
1060 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1061 {
1062 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1063 hmR0VmxVmcsInfoInit(&pVCpu->hmr0.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfo);
1064 hmR0VmxVmcsInfoInit(&pVCpu->hmr0.s.vmx.VmcsInfoNstGst, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
1065 }
1066}
1067
1068#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1069/**
1070 * Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1071 *
1072 * @returns @c true if the MSR is intercepted, @c false otherwise.
1073 * @param pbMsrBitmap The MSR bitmap.
1074 * @param offMsr The MSR byte offset.
1075 * @param iBit The bit offset from the byte offset.
1076 */
1077DECLINLINE(bool) hmR0VmxIsMsrBitSet(uint8_t const *pbMsrBitmap, uint16_t offMsr, int32_t iBit)
1078{
1079 Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1080 return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1081}
1082#endif
1083
1084/**
1085 * Sets the permission bits for the specified MSR in the given MSR bitmap.
1086 *
1087 * If the passed VMCS is a nested-guest VMCS, this function ensures that the
1088 * read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1089 * VMX execution of the nested-guest, only if nested-guest is also not intercepting
1090 * the read/write access of this MSR.
1091 *
1092 * @param pVCpu The cross context virtual CPU structure.
1093 * @param pVmcsInfo The VMCS info. object.
1094 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1095 * @param idMsr The MSR value.
1096 * @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1097 * include both a read -and- a write permission!
1098 *
1099 * @sa CPUMGetVmxMsrPermission.
1100 * @remarks Can be called with interrupts disabled.
1101 */
1102static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1103{
1104 uint8_t *pbMsrBitmap = (uint8_t *)pVmcsInfo->pvMsrBitmap;
1105 Assert(pbMsrBitmap);
1106 Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1107
1108 /*
1109 * MSR-bitmap Layout:
1110 * Byte index MSR range Interpreted as
1111 * 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1112 * 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1113 * 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1114 * 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1115 *
1116 * A bit corresponding to an MSR within the above range causes a VM-exit
1117 * if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1118 * the MSR range, it always cause a VM-exit.
1119 *
1120 * See Intel spec. 24.6.9 "MSR-Bitmap Address".
1121 */
1122 uint16_t const offBitmapRead = 0;
1123 uint16_t const offBitmapWrite = 0x800;
1124 uint16_t offMsr;
1125 int32_t iBit;
1126 if (idMsr <= UINT32_C(0x00001fff))
1127 {
1128 offMsr = 0;
1129 iBit = idMsr;
1130 }
1131 else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1132 {
1133 offMsr = 0x400;
1134 iBit = idMsr - UINT32_C(0xc0000000);
1135 }
1136 else
1137 AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1138
1139 /*
1140 * Set the MSR read permission.
1141 */
1142 uint16_t const offMsrRead = offBitmapRead + offMsr;
1143 Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1144 if (fMsrpm & VMXMSRPM_ALLOW_RD)
1145 {
1146#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1147 bool const fClear = !fIsNstGstVmcs ? true
1148 : !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, offMsrRead, iBit);
1149#else
1150 RT_NOREF2(pVCpu, fIsNstGstVmcs);
1151 bool const fClear = true;
1152#endif
1153 if (fClear)
1154 ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1155 }
1156 else
1157 ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1158
1159 /*
1160 * Set the MSR write permission.
1161 */
1162 uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1163 Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1164 if (fMsrpm & VMXMSRPM_ALLOW_WR)
1165 {
1166#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1167 bool const fClear = !fIsNstGstVmcs ? true
1168 : !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, offMsrWrite, iBit);
1169#else
1170 RT_NOREF2(pVCpu, fIsNstGstVmcs);
1171 bool const fClear = true;
1172#endif
1173 if (fClear)
1174 ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1175 }
1176 else
1177 ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1178}
1179
1180
1181/**
1182 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1183 * area.
1184 *
1185 * @returns VBox status code.
1186 * @param pVCpu The cross context virtual CPU structure.
1187 * @param pVmcsInfo The VMCS info. object.
1188 * @param cMsrs The number of MSRs.
1189 */
1190static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1191{
1192 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1193 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc);
1194 if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1195 {
1196 /* Commit the MSR counts to the VMCS and update the cache. */
1197 if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1198 {
1199 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
1200 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
1201 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
1202 pVmcsInfo->cEntryMsrLoad = cMsrs;
1203 pVmcsInfo->cExitMsrStore = cMsrs;
1204 pVmcsInfo->cExitMsrLoad = cMsrs;
1205 }
1206 return VINF_SUCCESS;
1207 }
1208
1209 LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1210 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1212}
1213
1214
1215/**
1216 * Adds a new (or updates the value of an existing) guest/host MSR
1217 * pair to be swapped during the world-switch as part of the
1218 * auto-load/store MSR area in the VMCS.
1219 *
1220 * @returns VBox status code.
1221 * @param pVCpu The cross context virtual CPU structure.
1222 * @param pVmxTransient The VMX-transient structure.
1223 * @param idMsr The MSR.
1224 * @param uGuestMsrValue Value of the guest MSR.
1225 * @param fSetReadWrite Whether to set the guest read/write access of this
1226 * MSR (thus not causing a VM-exit).
1227 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1228 * necessary.
1229 */
1230static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1231 bool fSetReadWrite, bool fUpdateHostMsr)
1232{
1233 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1234 bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1235 PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1236 uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1237 uint32_t i;
1238
1239 /* Paranoia. */
1240 Assert(pGuestMsrLoad);
1241
1242#ifndef DEBUG_bird
1243 LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGuestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1244#endif
1245
1246 /* Check if the MSR already exists in the VM-entry MSR-load area. */
1247 for (i = 0; i < cMsrs; i++)
1248 {
1249 if (pGuestMsrLoad[i].u32Msr == idMsr)
1250 break;
1251 }
1252
1253 bool fAdded = false;
1254 if (i == cMsrs)
1255 {
1256 /* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1257 ++cMsrs;
1258 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1259 AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1260
1261 /* Set the guest to read/write this MSR without causing VM-exits. */
1262 if ( fSetReadWrite
1263 && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1264 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1265
1266 Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1267 fAdded = true;
1268 }
1269
1270 /* Update the MSR value for the newly added or already existing MSR. */
1271 pGuestMsrLoad[i].u32Msr = idMsr;
1272 pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1273
1274 /* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1275 if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1276 {
1277 PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1278 pGuestMsrStore[i].u32Msr = idMsr;
1279 pGuestMsrStore[i].u64Value = uGuestMsrValue;
1280 }
1281
1282 /* Update the corresponding slot in the host MSR area. */
1283 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1284 Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1285 Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1286 pHostMsr[i].u32Msr = idMsr;
1287
1288 /*
1289 * Only if the caller requests to update the host MSR value AND we've newly added the
1290 * MSR to the host MSR area do we actually update the value. Otherwise, it will be
1291 * updated by hmR0VmxUpdateAutoLoadHostMsrs().
1292 *
1293 * We do this for performance reasons since reading MSRs may be quite expensive.
1294 */
1295 if (fAdded)
1296 {
1297 if (fUpdateHostMsr)
1298 {
1299 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1300 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1301 pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1302 }
1303 else
1304 {
1305 /* Someone else can do the work. */
1306 pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1307 }
1308 }
1309 return VINF_SUCCESS;
1310}
1311
1312
1313/**
1314 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1315 * auto-load/store MSR area in the VMCS.
1316 *
1317 * @returns VBox status code.
1318 * @param pVCpu The cross context virtual CPU structure.
1319 * @param pVmxTransient The VMX-transient structure.
1320 * @param idMsr The MSR.
1321 */
1322static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1323{
1324 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1325 bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1326 PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1327 uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1328
1329#ifndef DEBUG_bird
1330 LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1331#endif
1332
1333 for (uint32_t i = 0; i < cMsrs; i++)
1334 {
1335 /* Find the MSR. */
1336 if (pGuestMsrLoad[i].u32Msr == idMsr)
1337 {
1338 /*
1339 * If it's the last MSR, we only need to reduce the MSR count.
1340 * If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1341 */
1342 if (i < cMsrs - 1)
1343 {
1344 /* Remove it from the VM-entry MSR-load area. */
1345 pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1346 pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1347
1348 /* Remove it from the VM-exit MSR-store area if it's in a different page. */
1349 if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1350 {
1351 PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1352 Assert(pGuestMsrStore[i].u32Msr == idMsr);
1353 pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1354 pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1355 }
1356
1357 /* Remove it from the VM-exit MSR-load area. */
1358 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1359 Assert(pHostMsr[i].u32Msr == idMsr);
1360 pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1361 pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1362 }
1363
1364 /* Reduce the count to reflect the removed MSR and bail. */
1365 --cMsrs;
1366 break;
1367 }
1368 }
1369
1370 /* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1371 if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1372 {
1373 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1374 AssertRCReturn(rc, rc);
1375
1376 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1377 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1378 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR);
1379
1380 Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1381 return VINF_SUCCESS;
1382 }
1383
1384 return VERR_NOT_FOUND;
1385}
1386
1387
1388/**
1389 * Updates the value of all host MSRs in the VM-exit MSR-load area.
1390 *
1391 * @param pVCpu The cross context virtual CPU structure.
1392 * @param pVmcsInfo The VMCS info. object.
1393 *
1394 * @remarks No-long-jump zone!!!
1395 */
1396static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
1397{
1398 RT_NOREF(pVCpu);
1399 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1400
1401 PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1402 uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1403 Assert(pHostMsrLoad);
1404 Assert(sizeof(*pHostMsrLoad) * cMsrs <= X86_PAGE_4K_SIZE);
1405 LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1406 for (uint32_t i = 0; i < cMsrs; i++)
1407 {
1408 /*
1409 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1410 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1411 */
1412 if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1413 pHostMsrLoad[i].u64Value = g_uHmVmxHostMsrEfer;
1414 else
1415 pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1416 }
1417}
1418
1419
1420/**
1421 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1422 * perform lazy restoration of the host MSRs while leaving VT-x.
1423 *
1424 * @param pVCpu The cross context virtual CPU structure.
1425 *
1426 * @remarks No-long-jump zone!!!
1427 */
1428static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
1429{
1430 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1431
1432 /*
1433 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1434 */
1435 if (!(pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1436 {
1437 Assert(!(pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1438 if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fAllow64BitGuests)
1439 {
1440 pVCpu->hmr0.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1441 pVCpu->hmr0.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1442 pVCpu->hmr0.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1443 pVCpu->hmr0.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1444 }
1445 pVCpu->hmr0.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1446 }
1447}
1448
1449
1450#ifdef VBOX_STRICT
1451
1452/**
1453 * Verifies that our cached host EFER MSR value has not changed since we cached it.
1454 *
1455 * @param pVmcsInfo The VMCS info. object.
1456 */
1457static void hmR0VmxCheckHostEferMsr(PCVMXVMCSINFO pVmcsInfo)
1458{
1459 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1460
1461 if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1462 {
1463 uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
1464 uint64_t const uHostEferMsrCache = g_uHmVmxHostMsrEfer;
1465 uint64_t uVmcsEferMsrVmcs;
1466 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
1467 AssertRC(rc);
1468
1469 AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
1470 ("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
1471 AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
1472 ("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
1473 }
1474}
1475
1476
1477/**
1478 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1479 * VMCS are correct.
1480 *
1481 * @param pVCpu The cross context virtual CPU structure.
1482 * @param pVmcsInfo The VMCS info. object.
1483 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1484 */
1485static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1486{
1487 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1488
1489 /* Read the various MSR-area counts from the VMCS. */
1490 uint32_t cEntryLoadMsrs;
1491 uint32_t cExitStoreMsrs;
1492 uint32_t cExitLoadMsrs;
1493 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
1494 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
1495 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
1496
1497 /* Verify all the MSR counts are the same. */
1498 Assert(cEntryLoadMsrs == cExitStoreMsrs);
1499 Assert(cExitStoreMsrs == cExitLoadMsrs);
1500 uint32_t const cMsrs = cExitLoadMsrs;
1501
1502 /* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
1503 Assert(cMsrs < VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
1504
1505 /* Verify the MSR counts are within the allocated page size. */
1506 Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
1507
1508 /* Verify the relevant contents of the MSR areas match. */
1509 PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1510 PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1511 PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1512 bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1513 for (uint32_t i = 0; i < cMsrs; i++)
1514 {
1515 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1516 if (fSeparateExitMsrStorePage)
1517 {
1518 AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
1519 ("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
1520 pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
1521 }
1522
1523 AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
1524 ("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
1525 pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
1526
1527 uint64_t const u64HostMsr = ASMRdMsr(pHostMsrLoad->u32Msr);
1528 AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64HostMsr,
1529 ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1530 pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64HostMsr, cMsrs));
1531
1532 /* Verify that cached host EFER MSR matches what's loaded on the CPU. */
1533 bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
1534 AssertMsgReturnVoid(!fIsEferMsr || u64HostMsr == g_uHmVmxHostMsrEfer,
1535 ("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n", g_uHmVmxHostMsrEfer, u64HostMsr, cMsrs));
1536
1537 /* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
1538 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1539 {
1540 uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
1541 if (fIsEferMsr)
1542 {
1543 AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
1544 AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
1545 }
1546 else
1547 {
1548 /* Verify LBR MSRs (used only for debugging) are intercepted. We don't passthru these MSRs to the guest yet. */
1549 PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
1550 if ( pVM->hmr0.s.vmx.fLbr
1551 && ( hmR0VmxIsLbrBranchFromMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
1552 || hmR0VmxIsLbrBranchToMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
1553 || pGuestMsrLoad->u32Msr == pVM->hmr0.s.vmx.idLbrTosMsr))
1554 {
1555 AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_EXIT_RD_WR,
1556 ("u32Msr=%#RX32 cMsrs=%u Passthru read/write for LBR MSRs!\n",
1557 pGuestMsrLoad->u32Msr, cMsrs));
1558 }
1559 else if (!fIsNstGstVmcs)
1560 {
1561 AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_ALLOW_RD_WR,
1562 ("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
1563 }
1564 else
1565 {
1566 /*
1567 * A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
1568 * execute a nested-guest with MSR passthrough.
1569 *
1570 * Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
1571 * allow passthrough too.
1572 */
1573 void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap;
1574 Assert(pvMsrBitmapNstGst);
1575 uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
1576 AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
1577 ("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
1578 pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
1579 }
1580 }
1581 }
1582
1583 /* Move to the next MSR. */
1584 pHostMsrLoad++;
1585 pGuestMsrLoad++;
1586 pGuestMsrStore++;
1587 }
1588}
1589
1590#endif /* VBOX_STRICT */
1591
1592/**
1593 * Flushes the TLB using EPT.
1594 *
1595 * @returns VBox status code.
1596 * @param pVCpu The cross context virtual CPU structure of the calling
1597 * EMT. Can be NULL depending on @a enmTlbFlush.
1598 * @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
1599 * enmTlbFlush.
1600 * @param enmTlbFlush Type of flush.
1601 *
1602 * @remarks Caller is responsible for making sure this function is called only
1603 * when NestedPaging is supported and providing @a enmTlbFlush that is
1604 * supported by the CPU.
1605 * @remarks Can be called with interrupts disabled.
1606 */
1607static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
1608{
1609 uint64_t au64Descriptor[2];
1610 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1611 au64Descriptor[0] = 0;
1612 else
1613 {
1614 Assert(pVCpu);
1615 Assert(pVmcsInfo);
1616 au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
1617 }
1618 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1619
1620 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1621 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
1622
1623 if ( RT_SUCCESS(rc)
1624 && pVCpu)
1625 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1626}
1627
1628
1629/**
1630 * Flushes the TLB using VPID.
1631 *
1632 * @returns VBox status code.
1633 * @param pVCpu The cross context virtual CPU structure of the calling
1634 * EMT. Can be NULL depending on @a enmTlbFlush.
1635 * @param enmTlbFlush Type of flush.
1636 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1637 * on @a enmTlbFlush).
1638 *
1639 * @remarks Can be called with interrupts disabled.
1640 */
1641static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1642{
1643 Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fVpid);
1644
1645 uint64_t au64Descriptor[2];
1646 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1647 {
1648 au64Descriptor[0] = 0;
1649 au64Descriptor[1] = 0;
1650 }
1651 else
1652 {
1653 AssertPtr(pVCpu);
1654 AssertMsg(pVCpu->hmr0.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hmr0.s.uCurrentAsid));
1655 AssertMsg(pVCpu->hmr0.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hmr0.s.uCurrentAsid));
1656 au64Descriptor[0] = pVCpu->hmr0.s.uCurrentAsid;
1657 au64Descriptor[1] = GCPtr;
1658 }
1659
1660 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1661 AssertMsg(rc == VINF_SUCCESS,
1662 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hmr0.s.uCurrentAsid : 0, GCPtr, rc));
1663
1664 if ( RT_SUCCESS(rc)
1665 && pVCpu)
1666 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1667 NOREF(rc);
1668}
1669
1670
1671/**
1672 * Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
1673 * otherwise there is nothing really to invalidate.
1674 *
1675 * @returns VBox status code.
1676 * @param pVCpu The cross context virtual CPU structure.
1677 * @param GCVirt Guest virtual address of the page to invalidate.
1678 */
1679VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1680{
1681 AssertPtr(pVCpu);
1682 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1683
1684 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
1685 {
1686 /*
1687 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1688 * the EPT case. See @bugref{6043} and @bugref{6177}.
1689 *
1690 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1691 * as this function maybe called in a loop with individual addresses.
1692 */
1693 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1694 if (pVM->hmr0.s.vmx.fVpid)
1695 {
1696 if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1697 {
1698 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1699 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1700 }
1701 else
1702 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1703 }
1704 else if (pVM->hmr0.s.fNestedPaging)
1705 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1706 }
1707
1708 return VINF_SUCCESS;
1709}
1710
1711
1712/**
1713 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1714 * case where neither EPT nor VPID is supported by the CPU.
1715 *
1716 * @param pHostCpu The HM physical-CPU structure.
1717 * @param pVCpu The cross context virtual CPU structure.
1718 *
1719 * @remarks Called with interrupts disabled.
1720 */
1721static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
1722{
1723 AssertPtr(pVCpu);
1724 AssertPtr(pHostCpu);
1725
1726 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1727
1728 Assert(pHostCpu->idCpu != NIL_RTCPUID);
1729 pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1730 pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1731 pVCpu->hmr0.s.fForceTLBFlush = false;
1732 return;
1733}
1734
1735
1736/**
1737 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1738 *
1739 * @param pHostCpu The HM physical-CPU structure.
1740 * @param pVCpu The cross context virtual CPU structure.
1741 * @param pVmcsInfo The VMCS info. object.
1742 *
1743 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1744 * nomenclature. The reason is, to avoid confusion in compare statements
1745 * since the host-CPU copies are named "ASID".
1746 *
1747 * @remarks Called with interrupts disabled.
1748 */
1749static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
1750{
1751#ifdef VBOX_WITH_STATISTICS
1752 bool fTlbFlushed = false;
1753# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1754# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1755 if (!fTlbFlushed) \
1756 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1757 } while (0)
1758#else
1759# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1760# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1761#endif
1762
1763 AssertPtr(pVCpu);
1764 AssertPtr(pHostCpu);
1765 Assert(pHostCpu->idCpu != NIL_RTCPUID);
1766
1767 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1768 AssertMsg(pVM->hmr0.s.fNestedPaging && pVM->hmr0.s.vmx.fVpid,
1769 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1770 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hmr0.s.fNestedPaging, pVM->hmr0.s.vmx.fVpid));
1771
1772 /*
1773 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1774 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1775 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1776 * cannot reuse the current ASID anymore.
1777 */
1778 if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1779 || pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes)
1780 {
1781 ++pHostCpu->uCurrentAsid;
1782 if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1783 {
1784 pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1785 pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1786 pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1787 }
1788
1789 pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1790 pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1791 pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1792
1793 /*
1794 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1795 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1796 */
1797 hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hmr0.s.vmx.enmTlbFlushEpt);
1798 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1799 HMVMX_SET_TAGGED_TLB_FLUSHED();
1800 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1801 }
1802 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1803 {
1804 /*
1805 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1806 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1807 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1808 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1809 * mappings, see @bugref{6568}.
1810 *
1811 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1812 */
1813 hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hmr0.s.vmx.enmTlbFlushEpt);
1814 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1815 HMVMX_SET_TAGGED_TLB_FLUSHED();
1816 }
1817 else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
1818 {
1819 /*
1820 * The nested-guest specifies its own guest-physical address to use as the APIC-access
1821 * address which requires flushing the TLB of EPT cached structures.
1822 *
1823 * See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
1824 */
1825 hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hmr0.s.vmx.enmTlbFlushEpt);
1826 pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
1827 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
1828 HMVMX_SET_TAGGED_TLB_FLUSHED();
1829 }
1830
1831
1832 pVCpu->hmr0.s.fForceTLBFlush = false;
1833 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1834
1835 Assert(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu);
1836 Assert(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes);
1837 AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1838 ("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1839 AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1840 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
1841 pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hmr0.s.idLastCpu, pVCpu->hmr0.s.cTlbFlushes));
1842 AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1843 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1844
1845 /* Update VMCS with the VPID. */
1846 int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hmr0.s.uCurrentAsid);
1847 AssertRC(rc);
1848
1849#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1850}
1851
1852
1853/**
1854 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1855 *
1856 * @param pHostCpu The HM physical-CPU structure.
1857 * @param pVCpu The cross context virtual CPU structure.
1858 * @param pVmcsInfo The VMCS info. object.
1859 *
1860 * @remarks Called with interrupts disabled.
1861 */
1862static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
1863{
1864 AssertPtr(pVCpu);
1865 AssertPtr(pHostCpu);
1866 Assert(pHostCpu->idCpu != NIL_RTCPUID);
1867 AssertMsg(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
1868 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
1869
1870 /*
1871 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1872 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1873 */
1874 if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1875 || pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes)
1876 {
1877 pVCpu->hmr0.s.fForceTLBFlush = true;
1878 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1879 }
1880
1881 /* Check for explicit TLB flushes. */
1882 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1883 {
1884 pVCpu->hmr0.s.fForceTLBFlush = true;
1885 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1886 }
1887
1888 /* Check for TLB flushes while switching to/from a nested-guest. */
1889 if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
1890 {
1891 pVCpu->hmr0.s.fForceTLBFlush = true;
1892 pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
1893 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
1894 }
1895
1896 pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1897 pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1898
1899 if (pVCpu->hmr0.s.fForceTLBFlush)
1900 {
1901 hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.enmTlbFlushEpt);
1902 pVCpu->hmr0.s.fForceTLBFlush = false;
1903 }
1904}
1905
1906
1907/**
1908 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1909 *
1910 * @param pHostCpu The HM physical-CPU structure.
1911 * @param pVCpu The cross context virtual CPU structure.
1912 *
1913 * @remarks Called with interrupts disabled.
1914 */
1915static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
1916{
1917 AssertPtr(pVCpu);
1918 AssertPtr(pHostCpu);
1919 Assert(pHostCpu->idCpu != NIL_RTCPUID);
1920 AssertMsg(pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
1921 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
1922
1923 /*
1924 * Force a TLB flush for the first world switch if the current CPU differs from the one we
1925 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1926 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1927 * cannot reuse the current ASID anymore.
1928 */
1929 if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1930 || pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes)
1931 {
1932 pVCpu->hmr0.s.fForceTLBFlush = true;
1933 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1934 }
1935
1936 /* Check for explicit TLB flushes. */
1937 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1938 {
1939 /*
1940 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
1941 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
1942 * fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
1943 * include fExplicitFlush's too) - an obscure corner case.
1944 */
1945 pVCpu->hmr0.s.fForceTLBFlush = true;
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1947 }
1948
1949 /* Check for TLB flushes while switching to/from a nested-guest. */
1950 if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
1951 {
1952 pVCpu->hmr0.s.fForceTLBFlush = true;
1953 pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
1955 }
1956
1957 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1958 pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1959 if (pVCpu->hmr0.s.fForceTLBFlush)
1960 {
1961 ++pHostCpu->uCurrentAsid;
1962 if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1963 {
1964 pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1965 pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1966 pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1967 }
1968
1969 pVCpu->hmr0.s.fForceTLBFlush = false;
1970 pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1971 pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1972 if (pHostCpu->fFlushAsidBeforeUse)
1973 {
1974 if (pVM->hmr0.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
1975 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1976 else if (pVM->hmr0.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1977 {
1978 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
1979 pHostCpu->fFlushAsidBeforeUse = false;
1980 }
1981 else
1982 {
1983 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
1984 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
1985 }
1986 }
1987 }
1988
1989 AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1990 ("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1991 AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1992 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
1993 pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hmr0.s.idLastCpu, pVCpu->hmr0.s.cTlbFlushes));
1994 AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1995 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1996
1997 int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hmr0.s.uCurrentAsid);
1998 AssertRC(rc);
1999}
2000
2001
2002/**
2003 * Flushes the guest TLB entry based on CPU capabilities.
2004 *
2005 * @param pHostCpu The HM physical-CPU structure.
2006 * @param pVCpu The cross context virtual CPU structure.
2007 * @param pVmcsInfo The VMCS info. object.
2008 *
2009 * @remarks Called with interrupts disabled.
2010 */
2011static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2012{
2013#ifdef HMVMX_ALWAYS_FLUSH_TLB
2014 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2015#endif
2016 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2017 switch (pVM->hmr0.s.vmx.enmTlbFlushType)
2018 {
2019 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2020 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2021 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2022 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2023 default:
2024 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2025 break;
2026 }
2027 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2028}
2029
2030
2031/**
2032 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2033 * TLB entries from the host TLB before VM-entry.
2034 *
2035 * @returns VBox status code.
2036 * @param pVM The cross context VM structure.
2037 */
2038static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
2039{
2040 /*
2041 * Determine optimal flush type for nested paging.
2042 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2043 * unrestricted guest execution (see hmR3InitFinalizeR0()).
2044 */
2045 if (pVM->hmr0.s.fNestedPaging)
2046 {
2047 if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2048 {
2049 if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2050 pVM->hmr0.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2051 else if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2052 pVM->hmr0.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2053 else
2054 {
2055 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2056 pVM->hmr0.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2057 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2058 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2059 }
2060
2061 /* Make sure the write-back cacheable memory type for EPT is supported. */
2062 if (RT_UNLIKELY(!(g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_MEMTYPE_WB)))
2063 {
2064 pVM->hmr0.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2065 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2066 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2067 }
2068
2069 /* EPT requires a page-walk length of 4. */
2070 if (RT_UNLIKELY(!(g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2071 {
2072 pVM->hmr0.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2073 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2074 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2075 }
2076 }
2077 else
2078 {
2079 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2080 pVM->hmr0.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2081 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2082 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2083 }
2084 }
2085
2086 /*
2087 * Determine optimal flush type for VPID.
2088 */
2089 if (pVM->hmr0.s.vmx.fVpid)
2090 {
2091 if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2092 {
2093 if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2094 pVM->hmr0.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2095 else if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2096 pVM->hmr0.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2097 else
2098 {
2099 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2100 if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2101 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2102 if (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2103 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2104 pVM->hmr0.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2105 pVM->hmr0.s.vmx.fVpid = false;
2106 }
2107 }
2108 else
2109 {
2110 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2111 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2112 pVM->hmr0.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2113 pVM->hmr0.s.vmx.fVpid = false;
2114 }
2115 }
2116
2117 /*
2118 * Setup the handler for flushing tagged-TLBs.
2119 */
2120 if (pVM->hmr0.s.fNestedPaging && pVM->hmr0.s.vmx.fVpid)
2121 pVM->hmr0.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2122 else if (pVM->hmr0.s.fNestedPaging)
2123 pVM->hmr0.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2124 else if (pVM->hmr0.s.vmx.fVpid)
2125 pVM->hmr0.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2126 else
2127 pVM->hmr0.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2128
2129
2130 /*
2131 * Copy out the result to ring-3.
2132 */
2133 pVM->hm.s.ForR3.vmx.fVpid = pVM->hmr0.s.vmx.fVpid;
2134 pVM->hm.s.ForR3.vmx.enmTlbFlushType = pVM->hmr0.s.vmx.enmTlbFlushType;
2135 pVM->hm.s.ForR3.vmx.enmTlbFlushEpt = pVM->hmr0.s.vmx.enmTlbFlushEpt;
2136 pVM->hm.s.ForR3.vmx.enmTlbFlushVpid = pVM->hmr0.s.vmx.enmTlbFlushVpid;
2137 return VINF_SUCCESS;
2138}
2139
2140
2141/**
2142 * Sets up the LBR MSR ranges based on the host CPU.
2143 *
2144 * @returns VBox status code.
2145 * @param pVM The cross context VM structure.
2146 *
2147 * @sa nemR3DarwinSetupLbrMsrRange
2148 */
2149static int hmR0VmxSetupLbrMsrRange(PVMCC pVM)
2150{
2151 Assert(pVM->hmr0.s.vmx.fLbr);
2152 uint32_t idLbrFromIpMsrFirst;
2153 uint32_t idLbrFromIpMsrLast;
2154 uint32_t idLbrToIpMsrFirst;
2155 uint32_t idLbrToIpMsrLast;
2156 uint32_t idLbrTosMsr;
2157
2158 /*
2159 * Determine the LBR MSRs supported for this host CPU family and model.
2160 *
2161 * See Intel spec. 17.4.8 "LBR Stack".
2162 * See Intel "Model-Specific Registers" spec.
2163 */
2164 uint32_t const uFamilyModel = (g_CpumHostFeatures.s.uFamily << 8)
2165 | g_CpumHostFeatures.s.uModel;
2166 switch (uFamilyModel)
2167 {
2168 case 0x0f01: case 0x0f02:
2169 idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
2170 idLbrFromIpMsrLast = MSR_P4_LASTBRANCH_3;
2171 idLbrToIpMsrFirst = 0x0;
2172 idLbrToIpMsrLast = 0x0;
2173 idLbrTosMsr = MSR_P4_LASTBRANCH_TOS;
2174 break;
2175
2176 case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
2177 case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
2178 case 0x066a: case 0x066c: case 0x067d: case 0x067e:
2179 idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2180 idLbrFromIpMsrLast = MSR_LASTBRANCH_31_FROM_IP;
2181 idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2182 idLbrToIpMsrLast = MSR_LASTBRANCH_31_TO_IP;
2183 idLbrTosMsr = MSR_LASTBRANCH_TOS;
2184 break;
2185
2186 case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
2187 case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
2188 case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
2189 case 0x062e: case 0x0625: case 0x062c: case 0x062f:
2190 idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
2191 idLbrFromIpMsrLast = MSR_LASTBRANCH_15_FROM_IP;
2192 idLbrToIpMsrFirst = MSR_LASTBRANCH_0_TO_IP;
2193 idLbrToIpMsrLast = MSR_LASTBRANCH_15_TO_IP;
2194 idLbrTosMsr = MSR_LASTBRANCH_TOS;
2195 break;
2196
2197 case 0x0617: case 0x061d: case 0x060f:
2198 idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
2199 idLbrFromIpMsrLast = MSR_CORE2_LASTBRANCH_3_FROM_IP;
2200 idLbrToIpMsrFirst = MSR_CORE2_LASTBRANCH_0_TO_IP;
2201 idLbrToIpMsrLast = MSR_CORE2_LASTBRANCH_3_TO_IP;
2202 idLbrTosMsr = MSR_CORE2_LASTBRANCH_TOS;
2203 break;
2204
2205 /* Atom and related microarchitectures we don't care about:
2206 case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
2207 case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
2208 case 0x0636: */
2209 /* All other CPUs: */
2210 default:
2211 {
2212 LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
2213 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
2214 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2215 }
2216 }
2217
2218 /*
2219 * Validate.
2220 */
2221 uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
2222 PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
2223 AssertCompile( RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr)
2224 == RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrToIpMsr));
2225 if (cLbrStack > RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr))
2226 {
2227 LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
2228 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
2229 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2230 }
2231 NOREF(pVCpu0);
2232
2233 /*
2234 * Update the LBR info. to the VM struct. for use later.
2235 */
2236 pVM->hmr0.s.vmx.idLbrTosMsr = idLbrTosMsr;
2237
2238 pVM->hm.s.ForR3.vmx.idLbrFromIpMsrFirst = pVM->hmr0.s.vmx.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
2239 pVM->hm.s.ForR3.vmx.idLbrFromIpMsrLast = pVM->hmr0.s.vmx.idLbrFromIpMsrLast = idLbrFromIpMsrLast;
2240
2241 pVM->hm.s.ForR3.vmx.idLbrToIpMsrFirst = pVM->hmr0.s.vmx.idLbrToIpMsrFirst = idLbrToIpMsrFirst;
2242 pVM->hm.s.ForR3.vmx.idLbrToIpMsrLast = pVM->hmr0.s.vmx.idLbrToIpMsrLast = idLbrToIpMsrLast;
2243 return VINF_SUCCESS;
2244}
2245
2246
2247#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2248/**
2249 * Sets up the shadow VMCS fields arrays.
2250 *
2251 * This function builds arrays of VMCS fields to sync the shadow VMCS later while
2252 * executing the guest.
2253 *
2254 * @returns VBox status code.
2255 * @param pVM The cross context VM structure.
2256 */
2257static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
2258{
2259 /*
2260 * Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
2261 * when the host does not support it.
2262 */
2263 bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
2264 if ( !fGstVmwriteAll
2265 || (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL))
2266 { /* likely. */ }
2267 else
2268 {
2269 LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
2270 VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
2271 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272 }
2273
2274 uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
2275 uint32_t cRwFields = 0;
2276 uint32_t cRoFields = 0;
2277 for (uint32_t i = 0; i < cVmcsFields; i++)
2278 {
2279 VMXVMCSFIELD VmcsField;
2280 VmcsField.u = g_aVmcsFields[i];
2281
2282 /*
2283 * We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
2284 * Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
2285 * in the shadow VMCS fields array as they would be redundant.
2286 *
2287 * If the VMCS field depends on a CPU feature that is not exposed to the guest,
2288 * we must not include it in the shadow VMCS fields array. Guests attempting to
2289 * VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
2290 * the required behavior.
2291 */
2292 if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
2293 && CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
2294 {
2295 /*
2296 * Read-only fields are placed in a separate array so that while syncing shadow
2297 * VMCS fields later (which is more performance critical) we can avoid branches.
2298 *
2299 * However, if the guest can write to all fields (including read-only fields),
2300 * we treat it a as read/write field. Otherwise, writing to these fields would
2301 * cause a VMWRITE instruction error while syncing the shadow VMCS.
2302 */
2303 if ( fGstVmwriteAll
2304 || !VMXIsVmcsFieldReadOnly(VmcsField.u))
2305 pVM->hmr0.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
2306 else
2307 pVM->hmr0.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
2308 }
2309 }
2310
2311 /* Update the counts. */
2312 pVM->hmr0.s.vmx.cShadowVmcsFields = cRwFields;
2313 pVM->hmr0.s.vmx.cShadowVmcsRoFields = cRoFields;
2314 return VINF_SUCCESS;
2315}
2316
2317
2318/**
2319 * Sets up the VMREAD and VMWRITE bitmaps.
2320 *
2321 * @param pVM The cross context VM structure.
2322 */
2323static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
2324{
2325 /*
2326 * By default, ensure guest attempts to access any VMCS fields cause VM-exits.
2327 */
2328 uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
2329 uint8_t *pbVmreadBitmap = (uint8_t *)pVM->hmr0.s.vmx.pvVmreadBitmap;
2330 uint8_t *pbVmwriteBitmap = (uint8_t *)pVM->hmr0.s.vmx.pvVmwriteBitmap;
2331 ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
2332 ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
2333
2334 /*
2335 * Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
2336 * VMREAD and VMWRITE bitmaps.
2337 */
2338 {
2339 uint32_t const *paShadowVmcsFields = pVM->hmr0.s.vmx.paShadowVmcsFields;
2340 uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
2341 for (uint32_t i = 0; i < cShadowVmcsFields; i++)
2342 {
2343 uint32_t const uVmcsField = paShadowVmcsFields[i];
2344 Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
2345 Assert(uVmcsField >> 3 < cbBitmap);
2346 ASMBitClear(pbVmreadBitmap, uVmcsField & 0x7fff);
2347 ASMBitClear(pbVmwriteBitmap, uVmcsField & 0x7fff);
2348 }
2349 }
2350
2351 /*
2352 * Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
2353 * if the host supports VMWRITE to all supported VMCS fields.
2354 */
2355 if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
2356 {
2357 uint32_t const *paShadowVmcsRoFields = pVM->hmr0.s.vmx.paShadowVmcsRoFields;
2358 uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
2359 for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
2360 {
2361 uint32_t const uVmcsField = paShadowVmcsRoFields[i];
2362 Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
2363 Assert(uVmcsField >> 3 < cbBitmap);
2364 ASMBitClear(pbVmreadBitmap, uVmcsField & 0x7fff);
2365 }
2366 }
2367}
2368#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2369
2370
2371/**
2372 * Sets up the virtual-APIC page address for the VMCS.
2373 *
2374 * @param pVmcsInfo The VMCS info. object.
2375 */
2376DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
2377{
2378 RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2379 Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2380 Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2381 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2382 AssertRC(rc);
2383}
2384
2385
2386/**
2387 * Sets up the MSR-bitmap address for the VMCS.
2388 *
2389 * @param pVmcsInfo The VMCS info. object.
2390 */
2391DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
2392{
2393 RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2394 Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2395 Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2396 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2397 AssertRC(rc);
2398}
2399
2400
2401/**
2402 * Sets up the APIC-access page address for the VMCS.
2403 *
2404 * @param pVCpu The cross context virtual CPU structure.
2405 */
2406DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
2407{
2408 RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysApicAccess;
2409 Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2410 Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2411 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2412 AssertRC(rc);
2413}
2414
2415#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2416
2417/**
2418 * Sets up the VMREAD bitmap address for the VMCS.
2419 *
2420 * @param pVCpu The cross context virtual CPU structure.
2421 */
2422DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
2423{
2424 RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysVmreadBitmap;
2425 Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
2426 Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2427 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
2428 AssertRC(rc);
2429}
2430
2431
2432/**
2433 * Sets up the VMWRITE bitmap address for the VMCS.
2434 *
2435 * @param pVCpu The cross context virtual CPU structure.
2436 */
2437DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
2438{
2439 RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.HCPhysVmwriteBitmap;
2440 Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
2441 Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2442 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
2443 AssertRC(rc);
2444}
2445
2446#endif
2447
2448/**
2449 * Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2450 * in the VMCS.
2451 *
2452 * @returns VBox status code.
2453 * @param pVmcsInfo The VMCS info. object.
2454 */
2455DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
2456{
2457 RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2458 Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2459 Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2460
2461 RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2462 Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2463 Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2464
2465 RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2466 Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2467 Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2468
2469 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
2470 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
2471 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
2472 return VINF_SUCCESS;
2473}
2474
2475
2476/**
2477 * Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2478 *
2479 * @param pVCpu The cross context virtual CPU structure.
2480 * @param pVmcsInfo The VMCS info. object.
2481 */
2482static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2483{
2484 Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2485
2486 /*
2487 * By default, ensure guest attempts to access any MSR cause VM-exits.
2488 * This shall later be relaxed for specific MSRs as necessary.
2489 *
2490 * Note: For nested-guests, the entire bitmap will be merged prior to
2491 * executing the nested-guest using hardware-assisted VMX and hence there
2492 * is no need to perform this operation. See hmR0VmxMergeMsrBitmapNested.
2493 */
2494 Assert(pVmcsInfo->pvMsrBitmap);
2495 ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
2496
2497 /*
2498 * The guest can access the following MSRs (read, write) without causing
2499 * VM-exits; they are loaded/stored automatically using fields in the VMCS.
2500 */
2501 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2502 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2503 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2504 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2505 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2506 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2507
2508 /*
2509 * The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2510 * associated with then. We never need to intercept access (writes need to be
2511 * executed without causing a VM-exit, reads will #GP fault anyway).
2512 *
2513 * The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2514 * read/write them. We swap the guest/host MSR value using the
2515 * auto-load/store MSR area.
2516 */
2517 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2518 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2519 if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2520 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2521 if (pVM->cpum.ro.GuestFeatures.fIbrs)
2522 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2523
2524 /*
2525 * Allow full read/write access for the following MSRs (mandatory for VT-x)
2526 * required for 64-bit guests.
2527 */
2528 if (pVM->hmr0.s.fAllow64BitGuests)
2529 {
2530 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2531 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2532 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2533 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2534 }
2535
2536 /*
2537 * IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2538 */
2539#ifdef VBOX_STRICT
2540 Assert(pVmcsInfo->pvMsrBitmap);
2541 uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2542 Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2543#endif
2544}
2545
2546
2547/**
2548 * Sets up pin-based VM-execution controls in the VMCS.
2549 *
2550 * @returns VBox status code.
2551 * @param pVCpu The cross context virtual CPU structure.
2552 * @param pVmcsInfo The VMCS info. object.
2553 */
2554static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2555{
2556 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2557 uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
2558 uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2559
2560 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2561 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2562
2563 if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2564 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2565
2566 /* Enable the VMX-preemption timer. */
2567 if (pVM->hmr0.s.vmx.fUsePreemptTimer)
2568 {
2569 Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2570 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2571 }
2572
2573#if 0
2574 /* Enable posted-interrupt processing. */
2575 if (pVM->hm.s.fPostedIntrs)
2576 {
2577 Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2578 Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2579 fVal |= VMX_PIN_CTLS_POSTED_INT;
2580 }
2581#endif
2582
2583 if ((fVal & fZap) != fVal)
2584 {
2585 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2586 g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
2587 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2588 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2589 }
2590
2591 /* Commit it to the VMCS and update our cache. */
2592 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2593 AssertRC(rc);
2594 pVmcsInfo->u32PinCtls = fVal;
2595
2596 return VINF_SUCCESS;
2597}
2598
2599
2600/**
2601 * Sets up secondary processor-based VM-execution controls in the VMCS.
2602 *
2603 * @returns VBox status code.
2604 * @param pVCpu The cross context virtual CPU structure.
2605 * @param pVmcsInfo The VMCS info. object.
2606 */
2607static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2608{
2609 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2610 uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2611 uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2612
2613 /* WBINVD causes a VM-exit. */
2614 if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2615 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2616
2617 /* Enable EPT (aka nested-paging). */
2618 if (pVM->hmr0.s.fNestedPaging)
2619 fVal |= VMX_PROC_CTLS2_EPT;
2620
2621 /* Enable the INVPCID instruction if we expose it to the guest and is supported
2622 by the hardware. Without this, guest executing INVPCID would cause a #UD. */
2623 if ( pVM->cpum.ro.GuestFeatures.fInvpcid
2624 && (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
2625 fVal |= VMX_PROC_CTLS2_INVPCID;
2626
2627 /* Enable VPID. */
2628 if (pVM->hmr0.s.vmx.fVpid)
2629 fVal |= VMX_PROC_CTLS2_VPID;
2630
2631 /* Enable unrestricted guest execution. */
2632 if (pVM->hmr0.s.vmx.fUnrestrictedGuest)
2633 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2634
2635#if 0
2636 if (pVM->hm.s.fVirtApicRegs)
2637 {
2638 /* Enable APIC-register virtualization. */
2639 Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2640 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2641
2642 /* Enable virtual-interrupt delivery. */
2643 Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2644 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2645 }
2646#endif
2647
2648 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
2649 where the TPR shadow resides. */
2650 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2651 * done dynamically. */
2652 if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2653 {
2654 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2655 hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
2656 }
2657
2658 /* Enable the RDTSCP instruction if we expose it to the guest and is supported
2659 by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
2660 if ( pVM->cpum.ro.GuestFeatures.fRdTscP
2661 && (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
2662 fVal |= VMX_PROC_CTLS2_RDTSCP;
2663
2664 /* Enable Pause-Loop exiting. */
2665 if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
2666 && pVM->hm.s.vmx.cPleGapTicks
2667 && pVM->hm.s.vmx.cPleWindowTicks)
2668 {
2669 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2670
2671 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
2672 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
2673 }
2674
2675 if ((fVal & fZap) != fVal)
2676 {
2677 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2678 g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
2679 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2680 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2681 }
2682
2683 /* Commit it to the VMCS and update our cache. */
2684 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2685 AssertRC(rc);
2686 pVmcsInfo->u32ProcCtls2 = fVal;
2687
2688 return VINF_SUCCESS;
2689}
2690
2691
2692/**
2693 * Sets up processor-based VM-execution controls in the VMCS.
2694 *
2695 * @returns VBox status code.
2696 * @param pVCpu The cross context virtual CPU structure.
2697 * @param pVmcsInfo The VMCS info. object.
2698 */
2699static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2700{
2701 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2702 uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2703 uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2704
2705 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2706 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2707 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2708 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2709 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2710 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2711 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2712
2713 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2714 if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2715 || (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2716 {
2717 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2718 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2719 }
2720
2721 /* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2722 if (!pVM->hmr0.s.fNestedPaging)
2723 {
2724 Assert(!pVM->hmr0.s.vmx.fUnrestrictedGuest);
2725 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2726 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2727 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2728 }
2729
2730 /* Use TPR shadowing if supported by the CPU. */
2731 if ( PDMHasApic(pVM)
2732 && (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
2733 {
2734 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2735 /* CR8 writes cause a VM-exit based on TPR threshold. */
2736 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2737 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2738 hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
2739 }
2740 else
2741 {
2742 /* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
2743 invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
2744 if (pVM->hmr0.s.fAllow64BitGuests)
2745 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2746 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2747 }
2748
2749 /* Use MSR-bitmaps if supported by the CPU. */
2750 if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2751 {
2752 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2753 hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
2754 }
2755
2756 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2757 if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2758 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2759
2760 if ((fVal & fZap) != fVal)
2761 {
2762 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2763 g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2764 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2765 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2766 }
2767
2768 /* Commit it to the VMCS and update our cache. */
2769 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2770 AssertRC(rc);
2771 pVmcsInfo->u32ProcCtls = fVal;
2772
2773 /* Set up MSR permissions that don't change through the lifetime of the VM. */
2774 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2775 hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2776
2777 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2778 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2779 return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2780
2781 /* Sanity check, should not really happen. */
2782 if (RT_LIKELY(!pVM->hmr0.s.vmx.fUnrestrictedGuest))
2783 { /* likely */ }
2784 else
2785 {
2786 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2787 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2788 }
2789
2790 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2791 return VINF_SUCCESS;
2792}
2793
2794
2795/**
2796 * Sets up miscellaneous (everything other than Pin, Processor and secondary
2797 * Processor-based VM-execution) control fields in the VMCS.
2798 *
2799 * @returns VBox status code.
2800 * @param pVCpu The cross context virtual CPU structure.
2801 * @param pVmcsInfo The VMCS info. object.
2802 */
2803static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2804{
2805#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2806 if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUseVmcsShadowing)
2807 {
2808 hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
2809 hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
2810 }
2811#endif
2812
2813 Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
2814 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
2815 AssertRC(rc);
2816
2817 rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
2818 if (RT_SUCCESS(rc))
2819 {
2820 uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2821 uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2822
2823 rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2824 rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2825
2826 pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2827 pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2828
2829 if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fLbr)
2830 {
2831 rc = VMXWriteVmcsNw(VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2832 AssertRC(rc);
2833 }
2834 return VINF_SUCCESS;
2835 }
2836 else
2837 LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2838 return rc;
2839}
2840
2841
2842/**
2843 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2844 *
2845 * We shall setup those exception intercepts that don't change during the
2846 * lifetime of the VM here. The rest are done dynamically while loading the
2847 * guest state.
2848 *
2849 * @param pVCpu The cross context virtual CPU structure.
2850 * @param pVmcsInfo The VMCS info. object.
2851 */
2852static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2853{
2854 /*
2855 * The following exceptions are always intercepted:
2856 *
2857 * #AC - To prevent the guest from hanging the CPU and for dealing with
2858 * split-lock detecting host configs.
2859 * #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2860 * recursive #DBs can cause a CPU hang.
2861 * #PF - To sync our shadow page tables when nested-paging is not used.
2862 */
2863 bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2864 uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2865 | RT_BIT(X86_XCPT_DB)
2866 | (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
2867
2868 /* Commit it to the VMCS. */
2869 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2870 AssertRC(rc);
2871
2872 /* Update our cache of the exception bitmap. */
2873 pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2874}
2875
2876
2877#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2878/**
2879 * Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
2880 *
2881 * @returns VBox status code.
2882 * @param pVmcsInfo The VMCS info. object.
2883 */
2884static int hmR0VmxSetupVmcsCtlsNested(PVMXVMCSINFO pVmcsInfo)
2885{
2886 Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
2887 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
2888 AssertRC(rc);
2889
2890 rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
2891 if (RT_SUCCESS(rc))
2892 {
2893 if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2894 hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
2895
2896 /* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
2897 Assert(!pVmcsInfo->u64Cr0Mask);
2898 Assert(!pVmcsInfo->u64Cr4Mask);
2899 return VINF_SUCCESS;
2900 }
2901 LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
2902 return rc;
2903}
2904#endif
2905
2906
2907/**
2908 * Selector FNHMSVMVMRUN implementation.
2909 */
2910static DECLCALLBACK(int) hmR0VmxStartVmSelector(PVMXVMCSINFO pVmcsInfo, PVMCPUCC pVCpu, bool fResume)
2911{
2912 hmR0VmxUpdateStartVmFunction(pVCpu);
2913 return pVCpu->hmr0.s.vmx.pfnStartVm(pVmcsInfo, pVCpu, fResume);
2914}
2915
2916
2917/**
2918 * Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
2919 * VMX.
2920 *
2921 * @returns VBox status code.
2922 * @param pVCpu The cross context virtual CPU structure.
2923 * @param pVmcsInfo The VMCS info. object.
2924 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2925 */
2926static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2927{
2928 Assert(pVmcsInfo->pvVmcs);
2929 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2930
2931 /* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
2932 *(uint32_t *)pVmcsInfo->pvVmcs = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID);
2933 const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
2934
2935 LogFlowFunc(("\n"));
2936
2937 /*
2938 * Initialize the VMCS using VMCLEAR before loading the VMCS.
2939 * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
2940 */
2941 int rc = hmR0VmxClearVmcs(pVmcsInfo);
2942 if (RT_SUCCESS(rc))
2943 {
2944 rc = hmR0VmxLoadVmcs(pVmcsInfo);
2945 if (RT_SUCCESS(rc))
2946 {
2947 /*
2948 * Initialize the hardware-assisted VMX execution handler for guest and nested-guest VMCS.
2949 * The host is always 64-bit since we no longer support 32-bit hosts.
2950 * Currently we have just a single handler for all guest modes as well, see @bugref{6208#c73}.
2951 */
2952 if (!fIsNstGstVmcs)
2953 {
2954 rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
2955 if (RT_SUCCESS(rc))
2956 {
2957 rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
2958 if (RT_SUCCESS(rc))
2959 {
2960 rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
2961 if (RT_SUCCESS(rc))
2962 {
2963 hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
2964#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2965 /*
2966 * If a shadow VMCS is allocated for the VMCS info. object, initialize the
2967 * VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
2968 * making it fit for use when VMCS shadowing is later enabled.
2969 */
2970 if (pVmcsInfo->pvShadowVmcs)
2971 {
2972 VMXVMCSREVID VmcsRevId;
2973 VmcsRevId.u = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID);
2974 VmcsRevId.n.fIsShadowVmcs = 1;
2975 *(uint32_t *)pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
2976 rc = vmxHCClearShadowVmcs(pVmcsInfo);
2977 if (RT_SUCCESS(rc))
2978 { /* likely */ }
2979 else
2980 LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
2981 }
2982#endif
2983 }
2984 else
2985 LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
2986 }
2987 else
2988 LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
2989 }
2990 else
2991 LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
2992 }
2993 else
2994 {
2995#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2996 rc = hmR0VmxSetupVmcsCtlsNested(pVmcsInfo);
2997 if (RT_SUCCESS(rc))
2998 { /* likely */ }
2999 else
3000 LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3001#else
3002 AssertFailed();
3003#endif
3004 }
3005 }
3006 else
3007 LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3008 }
3009 else
3010 LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3011
3012 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
3013 if (RT_SUCCESS(rc))
3014 {
3015 rc = hmR0VmxClearVmcs(pVmcsInfo);
3016 if (RT_SUCCESS(rc))
3017 { /* likely */ }
3018 else
3019 LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3020 }
3021
3022 /*
3023 * Update the last-error record both for failures and success, so we
3024 * can propagate the status code back to ring-3 for diagnostics.
3025 */
3026 hmR0VmxUpdateErrorRecord(pVCpu, rc);
3027 NOREF(pszVmcs);
3028 return rc;
3029}
3030
3031
3032/**
3033 * Does global VT-x initialization (called during module initialization).
3034 *
3035 * @returns VBox status code.
3036 */
3037VMMR0DECL(int) VMXR0GlobalInit(void)
3038{
3039#ifdef HMVMX_USE_FUNCTION_TABLE
3040 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_aVMExitHandlers));
3041# ifdef VBOX_STRICT
3042 for (unsigned i = 0; i < RT_ELEMENTS(g_aVMExitHandlers); i++)
3043 Assert(g_aVMExitHandlers[i].pfn);
3044# endif
3045#endif
3046 return VINF_SUCCESS;
3047}
3048
3049
3050/**
3051 * Does global VT-x termination (called during module termination).
3052 */
3053VMMR0DECL(void) VMXR0GlobalTerm()
3054{
3055 /* Nothing to do currently. */
3056}
3057
3058
3059/**
3060 * Sets up and activates VT-x on the current CPU.
3061 *
3062 * @returns VBox status code.
3063 * @param pHostCpu The HM physical-CPU structure.
3064 * @param pVM The cross context VM structure. Can be
3065 * NULL after a host resume operation.
3066 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3067 * fEnabledByHost is @c true).
3068 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3069 * @a fEnabledByHost is @c true).
3070 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3071 * enable VT-x on the host.
3072 * @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3073 */
3074VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3075 PCSUPHWVIRTMSRS pHwvirtMsrs)
3076{
3077 AssertPtr(pHostCpu);
3078 AssertPtr(pHwvirtMsrs);
3079 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3080
3081 /* Enable VT-x if it's not already enabled by the host. */
3082 if (!fEnabledByHost)
3083 {
3084 int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
3085 if (RT_FAILURE(rc))
3086 return rc;
3087 }
3088
3089 /*
3090 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3091 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3092 * invalidated when flushing by VPID.
3093 */
3094 if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3095 {
3096 hmR0VmxFlushEpt(NULL /* pVCpu */, NULL /* pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3097 pHostCpu->fFlushAsidBeforeUse = false;
3098 }
3099 else
3100 pHostCpu->fFlushAsidBeforeUse = true;
3101
3102 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3103 ++pHostCpu->cTlbFlushes;
3104
3105 return VINF_SUCCESS;
3106}
3107
3108
3109/**
3110 * Deactivates VT-x on the current CPU.
3111 *
3112 * @returns VBox status code.
3113 * @param pHostCpu The HM physical-CPU structure.
3114 * @param pvCpuPage Pointer to the VMXON region.
3115 * @param HCPhysCpuPage Physical address of the VMXON region.
3116 *
3117 * @remarks This function should never be called when SUPR0EnableVTx() or
3118 * similar was used to enable VT-x on the host.
3119 */
3120VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3121{
3122 RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3123
3124 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3125 return hmR0VmxLeaveRootMode(pHostCpu);
3126}
3127
3128
3129/**
3130 * Does per-VM VT-x initialization.
3131 *
3132 * @returns VBox status code.
3133 * @param pVM The cross context VM structure.
3134 */
3135VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
3136{
3137 AssertPtr(pVM);
3138 LogFlowFunc(("pVM=%p\n", pVM));
3139
3140 hmR0VmxStructsInit(pVM);
3141 int rc = hmR0VmxStructsAlloc(pVM);
3142 if (RT_FAILURE(rc))
3143 {
3144 LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3145 return rc;
3146 }
3147
3148 /* Setup the crash dump page. */
3149#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3150 strcpy((char *)pVM->hmr0.s.vmx.pbScratch, "SCRATCH Magic");
3151 *(uint64_t *)(pVM->hmr0.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
3152#endif
3153 return VINF_SUCCESS;
3154}
3155
3156
3157/**
3158 * Does per-VM VT-x termination.
3159 *
3160 * @returns VBox status code.
3161 * @param pVM The cross context VM structure.
3162 */
3163VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
3164{
3165 AssertPtr(pVM);
3166 LogFlowFunc(("pVM=%p\n", pVM));
3167
3168#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3169 if (pVM->hmr0.s.vmx.pbScratch)
3170 RT_BZERO(pVM->hmr0.s.vmx.pbScratch, X86_PAGE_4K_SIZE);
3171#endif
3172 hmR0VmxStructsFree(pVM);
3173 return VINF_SUCCESS;
3174}
3175
3176
3177/**
3178 * Sets up the VM for execution using hardware-assisted VMX.
3179 * This function is only called once per-VM during initialization.
3180 *
3181 * @returns VBox status code.
3182 * @param pVM The cross context VM structure.
3183 */
3184VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
3185{
3186 AssertPtr(pVM);
3187 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3188
3189 LogFlowFunc(("pVM=%p\n", pVM));
3190
3191 /*
3192 * At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
3193 * without causing a #GP.
3194 */
3195 RTCCUINTREG const uHostCr4 = ASMGetCR4();
3196 if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
3197 { /* likely */ }
3198 else
3199 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3200
3201 /*
3202 * Check that nested paging is supported if enabled and copy over the flag to the
3203 * ring-0 only structure.
3204 */
3205 bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
3206 AssertReturn( !fNestedPaging
3207 || (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_EPT), /** @todo use a ring-0 copy of ProcCtls2.n.allowed1 */
3208 VERR_INCOMPATIBLE_CONFIG);
3209 pVM->hmr0.s.fNestedPaging = fNestedPaging;
3210 pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
3211
3212 /*
3213 * Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable *must*
3214 * always be allocated. We no longer support the highly unlikely case of unrestricted guest
3215 * without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3216 */
3217 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuestCfg;
3218 AssertReturn( !fUnrestrictedGuest
3219 || ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)
3220 && fNestedPaging),
3221 VERR_INCOMPATIBLE_CONFIG);
3222 if ( !fUnrestrictedGuest
3223 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3224 || !pVM->hm.s.vmx.pRealModeTSS))
3225 {
3226 LogRelFunc(("Invalid real-on-v86 state.\n"));
3227 return VERR_INTERNAL_ERROR;
3228 }
3229 pVM->hmr0.s.vmx.fUnrestrictedGuest = fUnrestrictedGuest;
3230
3231 /* Initialize these always, see hmR3InitFinalizeR0().*/
3232 pVM->hm.s.ForR3.vmx.enmTlbFlushEpt = pVM->hmr0.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3233 pVM->hm.s.ForR3.vmx.enmTlbFlushVpid = pVM->hmr0.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3234
3235 /* Setup the tagged-TLB flush handlers. */
3236 int rc = hmR0VmxSetupTaggedTlb(pVM);
3237 if (RT_FAILURE(rc))
3238 {
3239 LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
3240 return rc;
3241 }
3242
3243 /* Determine LBR capabilities. */
3244 pVM->hmr0.s.vmx.fLbr = pVM->hm.s.vmx.fLbrCfg;
3245 if (pVM->hmr0.s.vmx.fLbr)
3246 {
3247 rc = hmR0VmxSetupLbrMsrRange(pVM);
3248 if (RT_FAILURE(rc))
3249 {
3250 LogRelFunc(("Failed to setup LBR MSR range. rc=%Rrc\n", rc));
3251 return rc;
3252 }
3253 }
3254
3255#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3256 /* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
3257 if (pVM->hmr0.s.vmx.fUseVmcsShadowing)
3258 {
3259 rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
3260 if (RT_SUCCESS(rc))
3261 hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
3262 else
3263 {
3264 LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
3265 return rc;
3266 }
3267 }
3268#endif
3269
3270 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3271 {
3272 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
3273 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3274
3275 pVCpu->hmr0.s.vmx.pfnStartVm = hmR0VmxStartVmSelector;
3276
3277 rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hmr0.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3278 if (RT_SUCCESS(rc))
3279 {
3280#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3281 if (pVM->cpum.ro.GuestFeatures.fVmx)
3282 {
3283 rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hmr0.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3284 if (RT_SUCCESS(rc))
3285 { /* likely */ }
3286 else
3287 {
3288 LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3289 return rc;
3290 }
3291 }
3292#endif
3293 }
3294 else
3295 {
3296 LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3297 return rc;
3298 }
3299 }
3300
3301 return VINF_SUCCESS;
3302}
3303
3304
3305/**
3306 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3307 * the VMCS.
3308 * @returns CR4 for passing along to hmR0VmxExportHostSegmentRegs.
3309 */
3310static uint64_t hmR0VmxExportHostControlRegs(void)
3311{
3312 int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
3313 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
3314 uint64_t uHostCr4 = ASMGetCR4();
3315 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, uHostCr4); AssertRC(rc);
3316 return uHostCr4;
3317}
3318
3319
3320/**
3321 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3322 * the host-state area in the VMCS.
3323 *
3324 * @returns VBox status code.
3325 * @param pVCpu The cross context virtual CPU structure.
3326 * @param uHostCr4 The host CR4 value.
3327 */
3328static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu, uint64_t uHostCr4)
3329{
3330 /*
3331 * If we've executed guest code using hardware-assisted VMX, the host-state bits
3332 * will be messed up. We should -not- save the messed up state without restoring
3333 * the original host-state, see @bugref{7240}.
3334 *
3335 * This apparently can happen (most likely the FPU changes), deal with it rather than
3336 * asserting. Was observed booting Solaris 10u10 32-bit guest.
3337 */
3338 if (pVCpu->hmr0.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
3339 {
3340 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hmr0.s.vmx.fRestoreHostFlags,
3341 pVCpu->idCpu));
3342 VMXRestoreHostState(pVCpu->hmr0.s.vmx.fRestoreHostFlags, &pVCpu->hmr0.s.vmx.RestoreHost);
3343 pVCpu->hmr0.s.vmx.fRestoreHostFlags = 0;
3344 }
3345
3346 /*
3347 * Get all the host info.
3348 * ASSUME it is safe to use rdfsbase and friends if the CR4.FSGSBASE bit is set
3349 * without also checking the cpuid bit.
3350 */
3351 uint32_t fRestoreHostFlags;
3352#if RT_INLINE_ASM_EXTERNAL
3353 if (uHostCr4 & X86_CR4_FSGSBASE)
3354 {
3355 hmR0VmxExportHostSegmentRegsAsmHlp(&pVCpu->hmr0.s.vmx.RestoreHost, true /*fHaveFsGsBase*/);
3356 fRestoreHostFlags = VMX_RESTORE_HOST_CAN_USE_WRFSBASE_AND_WRGSBASE;
3357 }
3358 else
3359 {
3360 hmR0VmxExportHostSegmentRegsAsmHlp(&pVCpu->hmr0.s.vmx.RestoreHost, false /*fHaveFsGsBase*/);
3361 fRestoreHostFlags = 0;
3362 }
3363 RTSEL uSelES = pVCpu->hmr0.s.vmx.RestoreHost.uHostSelES;
3364 RTSEL uSelDS = pVCpu->hmr0.s.vmx.RestoreHost.uHostSelDS;
3365 RTSEL uSelFS = pVCpu->hmr0.s.vmx.RestoreHost.uHostSelFS;
3366 RTSEL uSelGS = pVCpu->hmr0.s.vmx.RestoreHost.uHostSelGS;
3367#else
3368 pVCpu->hmr0.s.vmx.RestoreHost.uHostSelTR = ASMGetTR();
3369 pVCpu->hmr0.s.vmx.RestoreHost.uHostSelSS = ASMGetSS();
3370 pVCpu->hmr0.s.vmx.RestoreHost.uHostSelCS = ASMGetCS();
3371 ASMGetGDTR((PRTGDTR)&pVCpu->hmr0.s.vmx.RestoreHost.HostGdtr);
3372 ASMGetIDTR((PRTIDTR)&pVCpu->hmr0.s.vmx.RestoreHost.HostIdtr);
3373 if (uHostCr4 & X86_CR4_FSGSBASE)
3374 {
3375 pVCpu->hmr0.s.vmx.RestoreHost.uHostFSBase = ASMGetFSBase();
3376 pVCpu->hmr0.s.vmx.RestoreHost.uHostGSBase = ASMGetGSBase();
3377 fRestoreHostFlags = VMX_RESTORE_HOST_CAN_USE_WRFSBASE_AND_WRGSBASE;
3378 }
3379 else
3380 {
3381 pVCpu->hmr0.s.vmx.RestoreHost.uHostFSBase = ASMRdMsr(MSR_K8_FS_BASE);
3382 pVCpu->hmr0.s.vmx.RestoreHost.uHostGSBase = ASMRdMsr(MSR_K8_GS_BASE);
3383 fRestoreHostFlags = 0;
3384 }
3385 RTSEL uSelES, uSelDS, uSelFS, uSelGS;
3386 pVCpu->hmr0.s.vmx.RestoreHost.uHostSelDS = uSelDS = ASMGetDS();
3387 pVCpu->hmr0.s.vmx.RestoreHost.uHostSelES = uSelES = ASMGetES();
3388 pVCpu->hmr0.s.vmx.RestoreHost.uHostSelFS = uSelFS = ASMGetFS();
3389 pVCpu->hmr0.s.vmx.RestoreHost.uHostSelGS = uSelGS = ASMGetGS();
3390#endif
3391
3392 /*
3393 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3394 * gain VM-entry and restore them before we get preempted.
3395 *
3396 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3397 */
3398 RTSEL const uSelAll = uSelFS | uSelGS | uSelES | uSelDS;
3399 if (uSelAll & (X86_SEL_RPL | X86_SEL_LDT))
3400 {
3401 if (!(uSelAll & X86_SEL_LDT))
3402 {
3403#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_uVmcsVar) \
3404 do { \
3405 (a_uVmcsVar) = pVCpu->hmr0.s.vmx.RestoreHost.uHostSel##a_Seg; \
3406 if ((a_uVmcsVar) & X86_SEL_RPL) \
3407 { \
3408 fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##a_Seg; \
3409 (a_uVmcsVar) = 0; \
3410 } \
3411 } while (0)
3412 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3413 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3414 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3415 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3416#undef VMXLOCAL_ADJUST_HOST_SEG
3417 }
3418 else
3419 {
3420#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_uVmcsVar) \
3421 do { \
3422 (a_uVmcsVar) = pVCpu->hmr0.s.vmx.RestoreHost.uHostSel##a_Seg; \
3423 if ((a_uVmcsVar) & (X86_SEL_RPL | X86_SEL_LDT)) \
3424 { \
3425 if (!((a_uVmcsVar) & X86_SEL_LDT)) \
3426 fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##a_Seg; \
3427 else \
3428 { \
3429 uint32_t const fAttr = ASMGetSegAttr(a_uVmcsVar); \
3430 if ((fAttr & X86_DESC_P) && fAttr != UINT32_MAX) \
3431 fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##a_Seg; \
3432 } \
3433 (a_uVmcsVar) = 0; \
3434 } \
3435 } while (0)
3436 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3437 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3438 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3439 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3440#undef VMXLOCAL_ADJUST_HOST_SEG
3441 }
3442 }
3443
3444 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3445 Assert(!(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelTR & X86_SEL_RPL)); Assert(!(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelTR & X86_SEL_LDT)); Assert(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelTR);
3446 Assert(!(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelCS & X86_SEL_RPL)); Assert(!(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelCS & X86_SEL_LDT)); Assert(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelCS);
3447 Assert(!(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelSS & X86_SEL_RPL)); Assert(!(pVCpu->hmr0.s.vmx.RestoreHost.uHostSelSS & X86_SEL_LDT));
3448 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3449 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3450 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3451 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3452
3453 /*
3454 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3455 * them to the maximum limit (0xffff) on every VM-exit.
3456 */
3457 if (pVCpu->hmr0.s.vmx.RestoreHost.HostGdtr.cb != 0xffff)
3458 fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3459
3460 /*
3461 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3462 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3463 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3464 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3465 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3466 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3467 * at 0xffff on hosts where we are sure it won't cause trouble.
3468 */
3469#if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3470 if (pVCpu->hmr0.s.vmx.RestoreHost.HostIdtr.cb < 0x0fff)
3471#else
3472 if (pVCpu->hmr0.s.vmx.RestoreHost.HostIdtr.cb != 0xffff)
3473#endif
3474 fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3475
3476 /*
3477 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3478 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3479 * RPL should be too in most cases.
3480 */
3481 RTSEL const uSelTR = pVCpu->hmr0.s.vmx.RestoreHost.uHostSelTR;
3482 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= pVCpu->hmr0.s.vmx.RestoreHost.HostGdtr.cb,
3483 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, pVCpu->hmr0.s.vmx.RestoreHost.HostGdtr.cb),
3484 VERR_VMX_INVALID_HOST_STATE);
3485
3486 PCX86DESCHC pDesc = (PCX86DESCHC)(pVCpu->hmr0.s.vmx.RestoreHost.HostGdtr.uAddr + (uSelTR & X86_SEL_MASK));
3487 uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
3488
3489 /*
3490 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3491 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3492 * restoration if the host has something else. Task switching is not supported in 64-bit
3493 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3494 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3495 *
3496 * [1] See Intel spec. 3.5 "System Descriptor Types".
3497 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3498 */
3499 Assert(pDesc->System.u4Type == 11);
3500 if ( pDesc->System.u16LimitLow != 0x67
3501 || pDesc->System.u4LimitHigh)
3502 {
3503 fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3504
3505 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3506 if (g_fHmHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3507 fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3508 if (g_fHmHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3509 {
3510 /* The GDT is read-only but the writable GDT is available. */
3511 fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3512 pVCpu->hmr0.s.vmx.RestoreHost.HostGdtrRw.cb = pVCpu->hmr0.s.vmx.RestoreHost.HostGdtr.cb;
3513 int rc = SUPR0GetCurrentGdtRw(&pVCpu->hmr0.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3514 AssertRCReturn(rc, rc);
3515 }
3516 }
3517
3518 pVCpu->hmr0.s.vmx.fRestoreHostFlags = fRestoreHostFlags;
3519
3520 /*
3521 * Do all the VMCS updates in one block to assist nested virtualization.
3522 */
3523 int rc;
3524 rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, pVCpu->hmr0.s.vmx.RestoreHost.uHostSelCS); AssertRC(rc);
3525 rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, pVCpu->hmr0.s.vmx.RestoreHost.uHostSelSS); AssertRC(rc);
3526 rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
3527 rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
3528 rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
3529 rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
3530 rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, pVCpu->hmr0.s.vmx.RestoreHost.uHostSelTR); AssertRC(rc);
3531 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, pVCpu->hmr0.s.vmx.RestoreHost.HostGdtr.uAddr); AssertRC(rc);
3532 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, pVCpu->hmr0.s.vmx.RestoreHost.HostIdtr.uAddr); AssertRC(rc);
3533 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase); AssertRC(rc);
3534 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, pVCpu->hmr0.s.vmx.RestoreHost.uHostFSBase); AssertRC(rc);
3535 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, pVCpu->hmr0.s.vmx.RestoreHost.uHostGSBase); AssertRC(rc);
3536
3537 return VINF_SUCCESS;
3538}
3539
3540
3541/**
3542 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3543 * host-state area of the VMCS.
3544 *
3545 * These MSRs will be automatically restored on the host after every successful
3546 * VM-exit.
3547 *
3548 * @param pVCpu The cross context virtual CPU structure.
3549 *
3550 * @remarks No-long-jump zone!!!
3551 */
3552static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
3553{
3554 AssertPtr(pVCpu);
3555
3556 /*
3557 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3558 * rather than swapping them on every VM-entry.
3559 */
3560 hmR0VmxLazySaveHostMsrs(pVCpu);
3561
3562 /*
3563 * Host Sysenter MSRs.
3564 */
3565 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
3566 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
3567 rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
3568
3569 /*
3570 * Host EFER MSR.
3571 *
3572 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3573 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3574 */
3575 if (g_fHmVmxSupportsVmcsEfer)
3576 {
3577 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, g_uHmVmxHostMsrEfer);
3578 AssertRC(rc);
3579 }
3580
3581 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3582 * hmR0VmxExportGuestEntryExitCtls(). */
3583}
3584
3585
3586/**
3587 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3588 *
3589 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3590 * these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
3591 *
3592 * @returns true if we need to load guest EFER, false otherwise.
3593 * @param pVCpu The cross context virtual CPU structure.
3594 * @param pVmxTransient The VMX-transient structure.
3595 *
3596 * @remarks Requires EFER, CR4.
3597 * @remarks No-long-jump zone!!!
3598 */
3599static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3600{
3601#ifdef HMVMX_ALWAYS_SWAP_EFER
3602 RT_NOREF2(pVCpu, pVmxTransient);
3603 return true;
3604#else
3605 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3606 uint64_t const u64HostEfer = g_uHmVmxHostMsrEfer;
3607 uint64_t const u64GuestEfer = pCtx->msrEFER;
3608
3609# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3610 /*
3611 * For nested-guests, we shall honor swapping the EFER MSR when requested by
3612 * the nested-guest.
3613 */
3614 if ( pVmxTransient->fIsNestedGuest
3615 && ( CPUMIsGuestVmxEntryCtlsSet(pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3616 || CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
3617 || CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
3618 return true;
3619# else
3620 RT_NOREF(pVmxTransient);
3621#endif
3622
3623 /*
3624 * For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
3625 * to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3626 */
3627 if ( CPUMIsGuestInLongModeEx(pCtx)
3628 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3629 return true;
3630
3631 /*
3632 * If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
3633 * as it affects guest paging. 64-bit paging implies CR4.PAE as well.
3634 *
3635 * See Intel spec. 4.5 "IA-32e Paging".
3636 * See Intel spec. 4.1.1 "Three Paging Modes".
3637 *
3638 * Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
3639 * import CR4 and CR0 from the VMCS here as those bits are always up to date.
3640 */
3641 Assert(vmxHCGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
3642 Assert(vmxHCGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
3643 if ( (pCtx->cr4 & X86_CR4_PAE)
3644 && (pCtx->cr0 & X86_CR0_PG))
3645 {
3646 /*
3647 * If nested paging is not used, verify that the guest paging mode matches the
3648 * shadow paging mode which is/will be placed in the VMCS (which is what will
3649 * actually be used while executing the guest and not the CR4 shadow value).
3650 */
3651 AssertMsg( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3652 || pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
3653 || pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
3654 || pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
3655 || pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX,
3656 ("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
3657 if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3658 {
3659 /* Verify that the host is NX capable. */
3660 Assert(g_CpumHostFeatures.s.fNoExecute);
3661 return true;
3662 }
3663 }
3664
3665 return false;
3666#endif
3667}
3668
3669
3670/**
3671 * Exports the guest's RSP into the guest-state area in the VMCS.
3672 *
3673 * @param pVCpu The cross context virtual CPU structure.
3674 *
3675 * @remarks No-long-jump zone!!!
3676 */
3677static void hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
3678{
3679 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3680 {
3681 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3682
3683 int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3684 AssertRC(rc);
3685
3686 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3687 Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
3688 }
3689}
3690
3691
3692/**
3693 * Exports the guest hardware-virtualization state.
3694 *
3695 * @returns VBox status code.
3696 * @param pVCpu The cross context virtual CPU structure.
3697 * @param pVmxTransient The VMX-transient structure.
3698 *
3699 * @remarks No-long-jump zone!!!
3700 */
3701static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3702{
3703 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
3704 {
3705#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3706 /*
3707 * Check if the VMX feature is exposed to the guest and if the host CPU supports
3708 * VMCS shadowing.
3709 */
3710 if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUseVmcsShadowing)
3711 {
3712 /*
3713 * If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
3714 * copy the nested hypervisor's current VMCS into the shadow VMCS and enable
3715 * VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
3716 *
3717 * We check for VMX root mode here in case the guest executes VMXOFF without
3718 * clearing the current VMCS pointer and our VMXOFF instruction emulation does
3719 * not clear the current VMCS pointer.
3720 */
3721 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3722 if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
3723 && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
3724 && CPUMIsGuestVmxCurrentVmcsValid(&pVCpu->cpum.GstCtx))
3725 {
3726 /* Paranoia. */
3727 Assert(!pVmxTransient->fIsNestedGuest);
3728
3729 /*
3730 * For performance reasons, also check if the nested hypervisor's current VMCS
3731 * was newly loaded or modified before copying it to the shadow VMCS.
3732 */
3733 if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
3734 {
3735 int rc = vmxHCCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
3736 AssertRCReturn(rc, rc);
3737 pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
3738 }
3739 vmxHCEnableVmcsShadowing(pVCpu, pVmcsInfo);
3740 }
3741 else
3742 vmxHCDisableVmcsShadowing(pVCpu, pVmcsInfo);
3743 }
3744#else
3745 NOREF(pVmxTransient);
3746#endif
3747 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
3748 }
3749 return VINF_SUCCESS;
3750}
3751
3752
3753/**
3754 * Exports the guest debug registers into the guest-state area in the VMCS.
3755 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3756 *
3757 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
3758 *
3759 * @returns VBox status code.
3760 * @param pVCpu The cross context virtual CPU structure.
3761 * @param pVmxTransient The VMX-transient structure.
3762 *
3763 * @remarks No-long-jump zone!!!
3764 */
3765static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
3766{
3767 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3768
3769 /** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
3770 * stepping. */
3771 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
3772 if (pVmxTransient->fIsNestedGuest)
3773 {
3774 int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
3775 AssertRC(rc);
3776
3777 /*
3778 * We don't want to always intercept MOV DRx for nested-guests as it causes
3779 * problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
3780 * Instead, they are strictly only requested when the nested hypervisor intercepts
3781 * them -- handled while merging VMCS controls.
3782 *
3783 * If neither the outer nor the nested-hypervisor is intercepting MOV DRx,
3784 * then the nested-guest debug state should be actively loaded on the host so that
3785 * nested-guest reads its own debug registers without causing VM-exits.
3786 */
3787 if ( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT)
3788 && !CPUMIsGuestDebugStateActive(pVCpu))
3789 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3790 return VINF_SUCCESS;
3791 }
3792
3793#ifdef VBOX_STRICT
3794 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3795 if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
3796 {
3797 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3798 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
3799 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
3800 }
3801#endif
3802
3803 bool fSteppingDB = false;
3804 bool fInterceptMovDRx = false;
3805 uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
3806 if (pVCpu->hm.s.fSingleInstruction)
3807 {
3808 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3809 if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
3810 {
3811 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
3812 Assert(fSteppingDB == false);
3813 }
3814 else
3815 {
3816 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
3817 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
3818 pVCpu->hmr0.s.fClearTrapFlag = true;
3819 fSteppingDB = true;
3820 }
3821 }
3822
3823 uint64_t u64GuestDr7;
3824 if ( fSteppingDB
3825 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3826 {
3827 /*
3828 * Use the combined guest and host DRx values found in the hypervisor register set
3829 * because the hypervisor debugger has breakpoints active or someone is single stepping
3830 * on the host side without a monitor trap flag.
3831 *
3832 * Note! DBGF expects a clean DR6 state before executing guest code.
3833 */
3834 if (!CPUMIsHyperDebugStateActive(pVCpu))
3835 {
3836 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3837 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3838 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3839 }
3840
3841 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
3842 u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
3843 pVCpu->hmr0.s.fUsingHyperDR7 = true;
3844 fInterceptMovDRx = true;
3845 }
3846 else
3847 {
3848 /*
3849 * If the guest has enabled debug registers, we need to load them prior to
3850 * executing guest code so they'll trigger at the right time.
3851 */
3852 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
3853 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
3854 {
3855 if (!CPUMIsGuestDebugStateActive(pVCpu))
3856 {
3857 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3858 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3859 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3860 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3861 }
3862 Assert(!fInterceptMovDRx);
3863 }
3864 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3865 {
3866 /*
3867 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3868 * must intercept #DB in order to maintain a correct DR6 guest value, and
3869 * because we need to intercept it to prevent nested #DBs from hanging the
3870 * CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
3871 */
3872 fInterceptMovDRx = true;
3873 }
3874
3875 /* Update DR7 with the actual guest value. */
3876 u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
3877 pVCpu->hmr0.s.fUsingHyperDR7 = false;
3878 }
3879
3880 if (fInterceptMovDRx)
3881 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
3882 else
3883 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
3884
3885 /*
3886 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
3887 * monitor-trap flag and update our cache.
3888 */
3889 if (uProcCtls != pVmcsInfo->u32ProcCtls)
3890 {
3891 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3892 AssertRC(rc);
3893 pVmcsInfo->u32ProcCtls = uProcCtls;
3894 }
3895
3896 /*
3897 * Update guest DR7.
3898 */
3899 int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
3900 AssertRC(rc);
3901
3902 /*
3903 * If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
3904 * we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
3905 *
3906 * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
3907 */
3908 if (fSteppingDB)
3909 {
3910 Assert(pVCpu->hm.s.fSingleInstruction);
3911 Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
3912
3913 uint32_t fIntrState = 0;
3914 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
3915 AssertRC(rc);
3916
3917 if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
3918 {
3919 fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
3920 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3921 AssertRC(rc);
3922 }
3923 }
3924
3925 return VINF_SUCCESS;
3926}
3927
3928
3929/**
3930 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3931 * areas.
3932 *
3933 * These MSRs will automatically be loaded to the host CPU on every successful
3934 * VM-entry and stored from the host CPU on every successful VM-exit.
3935 *
3936 * We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
3937 * actual host MSR values are not- updated here for performance reasons. See
3938 * hmR0VmxExportHostMsrs().
3939 *
3940 * We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
3941 *
3942 * @returns VBox status code.
3943 * @param pVCpu The cross context virtual CPU structure.
3944 * @param pVmxTransient The VMX-transient structure.
3945 *
3946 * @remarks No-long-jump zone!!!
3947 */
3948static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
3949{
3950 AssertPtr(pVCpu);
3951 AssertPtr(pVmxTransient);
3952
3953 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3954 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3955
3956 /*
3957 * MSRs that we use the auto-load/store MSR area in the VMCS.
3958 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
3959 * nothing to do here. The host MSR values are updated when it's safe in
3960 * hmR0VmxLazySaveHostMsrs().
3961 *
3962 * For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
3963 * loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
3964 * emulation. The merged MSR permission bitmap will ensure that we get VM-exits
3965 * for any MSR that are not part of the lazy MSRs so we do not need to place
3966 * those MSRs into the auto-load/store MSR area. Nothing to do here.
3967 */
3968 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3969 {
3970 /* No auto-load/store MSRs currently. */
3971 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
3972 }
3973
3974 /*
3975 * Guest Sysenter MSRs.
3976 */
3977 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
3978 {
3979 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
3980
3981 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3982 {
3983 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3984 AssertRC(rc);
3985 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
3986 }
3987
3988 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3989 {
3990 int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3991 AssertRC(rc);
3992 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
3993 }
3994
3995 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3996 {
3997 int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3998 AssertRC(rc);
3999 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4000 }
4001 }
4002
4003 /*
4004 * Guest/host EFER MSR.
4005 */
4006 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4007 {
4008 /* Whether we are using the VMCS to swap the EFER MSR must have been
4009 determined earlier while exporting VM-entry/VM-exit controls. */
4010 Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
4011 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4012
4013 if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4014 {
4015 /*
4016 * EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
4017 * This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
4018 * VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
4019 * and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
4020 * must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
4021 * the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
4022 * during VM-entry.
4023 */
4024 uint64_t uGuestEferMsr = pCtx->msrEFER;
4025 if (!pVM->hmr0.s.vmx.fUnrestrictedGuest)
4026 {
4027 if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
4028 uGuestEferMsr &= ~MSR_K6_EFER_LME;
4029 else
4030 Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
4031 }
4032
4033 /*
4034 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4035 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4036 */
4037 if (g_fHmVmxSupportsVmcsEfer)
4038 {
4039 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
4040 AssertRC(rc);
4041 }
4042 else
4043 {
4044 /*
4045 * We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
4046 * continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
4047 */
4048 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
4049 false /* fSetReadWrite */, false /* fUpdateHostMsr */);
4050 AssertRCReturn(rc, rc);
4051 }
4052
4053 Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
4054 }
4055 else if (!g_fHmVmxSupportsVmcsEfer)
4056 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
4057
4058 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4059 }
4060
4061 /*
4062 * Other MSRs.
4063 */
4064 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
4065 {
4066 /* Speculation Control (R/W). */
4067 HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
4068 if (pVM->cpum.ro.GuestFeatures.fIbrs)
4069 {
4070 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
4071 false /* fSetReadWrite */, false /* fUpdateHostMsr */);
4072 AssertRCReturn(rc, rc);
4073 }
4074
4075 /* Last Branch Record. */
4076 if (pVM->hmr0.s.vmx.fLbr)
4077 {
4078 PVMXVMCSINFOSHARED const pVmcsInfoShared = pVmxTransient->pVmcsInfo->pShared;
4079 uint32_t const idFromIpMsrStart = pVM->hmr0.s.vmx.idLbrFromIpMsrFirst;
4080 uint32_t const idToIpMsrStart = pVM->hmr0.s.vmx.idLbrToIpMsrFirst;
4081 uint32_t const cLbrStack = pVM->hmr0.s.vmx.idLbrFromIpMsrLast - pVM->hmr0.s.vmx.idLbrFromIpMsrFirst + 1;
4082 Assert(cLbrStack <= 32);
4083 for (uint32_t i = 0; i < cLbrStack; i++)
4084 {
4085 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idFromIpMsrStart + i,
4086 pVmcsInfoShared->au64LbrFromIpMsr[i],
4087 false /* fSetReadWrite */, false /* fUpdateHostMsr */);
4088 AssertRCReturn(rc, rc);
4089
4090 /* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
4091 if (idToIpMsrStart != 0)
4092 {
4093 rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idToIpMsrStart + i,
4094 pVmcsInfoShared->au64LbrToIpMsr[i],
4095 false /* fSetReadWrite */, false /* fUpdateHostMsr */);
4096 AssertRCReturn(rc, rc);
4097 }
4098 }
4099
4100 /* Add LBR top-of-stack MSR (which contains the index to the most recent record). */
4101 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, pVM->hmr0.s.vmx.idLbrTosMsr,
4102 pVmcsInfoShared->u64LbrTosMsr, false /* fSetReadWrite */,
4103 false /* fUpdateHostMsr */);
4104 AssertRCReturn(rc, rc);
4105 }
4106
4107 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
4108 }
4109
4110 return VINF_SUCCESS;
4111}
4112
4113
4114/**
4115 * Wrapper for running the guest code in VT-x.
4116 *
4117 * @returns VBox status code, no informational status codes.
4118 * @param pVCpu The cross context virtual CPU structure.
4119 * @param pVmxTransient The VMX-transient structure.
4120 *
4121 * @remarks No-long-jump zone!!!
4122 */
4123DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4124{
4125 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4126 pVCpu->cpum.GstCtx.fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4127
4128 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4129 bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
4130#ifdef VBOX_WITH_STATISTICS
4131 if (fResumeVM)
4132 STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxVmResume);
4133 else
4134 STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxVmLaunch);
4135#endif
4136 int rc = pVCpu->hmr0.s.vmx.pfnStartVm(pVmcsInfo, pVCpu, fResumeVM);
4137 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4138 return rc;
4139}
4140
4141
4142/**
4143 * Reports world-switch error and dumps some useful debug info.
4144 *
4145 * @param pVCpu The cross context virtual CPU structure.
4146 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4147 * @param pVmxTransient The VMX-transient structure (only
4148 * exitReason updated).
4149 */
4150static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4151{
4152 Assert(pVCpu);
4153 Assert(pVmxTransient);
4154 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4155
4156 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4157 switch (rcVMRun)
4158 {
4159 case VERR_VMX_INVALID_VMXON_PTR:
4160 AssertFailed();
4161 break;
4162 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4163 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4164 {
4165 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4166 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4167 AssertRC(rc);
4168 vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
4169
4170 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hmr0.s.idEnteredCpu;
4171 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4172 Cannot do it here as we may have been long preempted. */
4173
4174#ifdef VBOX_STRICT
4175 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4176 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4177 pVmxTransient->uExitReason));
4178 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
4179 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4180 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4181 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4182 else
4183 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4184 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4185 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4186
4187 static struct
4188 {
4189 /** Name of the field to log. */
4190 const char *pszName;
4191 /** The VMCS field. */
4192 uint32_t uVmcsField;
4193 /** Whether host support of this field needs to be checked. */
4194 bool fCheckSupport;
4195 } const s_aVmcsFields[] =
4196 {
4197 { "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
4198 { "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
4199 { "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
4200 { "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
4201 { "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
4202 { "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
4203 { "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
4204 { "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
4205 { "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
4206 { "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
4207 { "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
4208 { "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
4209 { "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
4210 { "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
4211 { "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
4212 { "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
4213 { "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
4214 { "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
4215 { "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
4216 { "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
4217 { "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
4218 { "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
4219 { "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
4220 { "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
4221 { "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
4222 { "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
4223 { "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
4224 { "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
4225 /* The order of selector fields below are fixed! */
4226 { "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
4227 { "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
4228 { "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
4229 { "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
4230 { "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
4231 { "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
4232 { "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
4233 /* End of ordered selector fields. */
4234 { "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
4235 { "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
4236 { "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
4237 { "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
4238 { "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
4239 { "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
4240 { "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
4241 { "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
4242 };
4243
4244 RTGDTR HostGdtr;
4245 ASMGetGDTR(&HostGdtr);
4246
4247 uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
4248 for (uint32_t i = 0; i < cVmcsFields; i++)
4249 {
4250 uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
4251
4252 bool fSupported;
4253 if (!s_aVmcsFields[i].fCheckSupport)
4254 fSupported = true;
4255 else
4256 {
4257 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4258 switch (uVmcsField)
4259 {
4260 case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hmr0.s.fNestedPaging; break;
4261 case VMX_VMCS16_VPID: fSupported = pVM->hmr0.s.vmx.fVpid; break;
4262 case VMX_VMCS32_CTRL_PROC_EXEC2:
4263 fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
4264 break;
4265 default:
4266 AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
4267 }
4268 }
4269
4270 if (fSupported)
4271 {
4272 uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
4273 switch (uWidth)
4274 {
4275 case VMX_VMCSFIELD_WIDTH_16BIT:
4276 {
4277 uint16_t u16Val;
4278 rc = VMXReadVmcs16(uVmcsField, &u16Val);
4279 AssertRC(rc);
4280 Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
4281
4282 if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
4283 && uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
4284 {
4285 if (u16Val < HostGdtr.cbGdt)
4286 {
4287 /* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
4288 static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
4289 "Host FS", "Host GS", "Host TR" };
4290 uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
4291 Assert(idxSel < RT_ELEMENTS(s_apszSel));
4292 PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
4293 hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
4294 }
4295 else
4296 Log4((" Selector value exceeds GDT limit!\n"));
4297 }
4298 break;
4299 }
4300
4301 case VMX_VMCSFIELD_WIDTH_32BIT:
4302 {
4303 uint32_t u32Val;
4304 rc = VMXReadVmcs32(uVmcsField, &u32Val);
4305 AssertRC(rc);
4306 Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
4307 break;
4308 }
4309
4310 case VMX_VMCSFIELD_WIDTH_64BIT:
4311 case VMX_VMCSFIELD_WIDTH_NATURAL:
4312 {
4313 uint64_t u64Val;
4314 rc = VMXReadVmcs64(uVmcsField, &u64Val);
4315 AssertRC(rc);
4316 Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
4317 break;
4318 }
4319 }
4320 }
4321 }
4322
4323 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4324 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4325 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4326 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4327 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4328 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4329#endif /* VBOX_STRICT */
4330 break;
4331 }
4332
4333 default:
4334 /* Impossible */
4335 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4336 break;
4337 }
4338}
4339
4340
4341/**
4342 * Sets up the usage of TSC-offsetting and updates the VMCS.
4343 *
4344 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
4345 * VMX-preemption timer.
4346 *
4347 * @returns VBox status code.
4348 * @param pVCpu The cross context virtual CPU structure.
4349 * @param pVmxTransient The VMX-transient structure.
4350 * @param idCurrentCpu The current CPU number.
4351 *
4352 * @remarks No-long-jump zone!!!
4353 */
4354static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, RTCPUID idCurrentCpu)
4355{
4356 bool fOffsettedTsc;
4357 bool fParavirtTsc;
4358 uint64_t uTscOffset;
4359 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4360 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4361
4362 if (pVM->hmr0.s.vmx.fUsePreemptTimer)
4363 {
4364 /* The TMCpuTickGetDeadlineAndTscOffset function is expensive (calling it on
4365 every entry slowed down the bs2-test1 CPUID testcase by ~33% (on an 10980xe). */
4366 uint64_t cTicksToDeadline;
4367 if ( idCurrentCpu == pVCpu->hmr0.s.idLastCpu
4368 && TMVirtualSyncIsCurrentDeadlineVersion(pVM, pVCpu->hmr0.s.vmx.uTscDeadlineVersion))
4369 {
4370 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatVmxPreemptionReusingDeadline);
4371 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
4372 cTicksToDeadline = pVCpu->hmr0.s.vmx.uTscDeadline - SUPReadTsc();
4373 if ((int64_t)cTicksToDeadline > 0)
4374 { /* hopefully */ }
4375 else
4376 {
4377 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatVmxPreemptionReusingDeadlineExpired);
4378 cTicksToDeadline = 0;
4379 }
4380 }
4381 else
4382 {
4383 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatVmxPreemptionRecalcingDeadline);
4384 cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc,
4385 &pVCpu->hmr0.s.vmx.uTscDeadline,
4386 &pVCpu->hmr0.s.vmx.uTscDeadlineVersion);
4387 pVCpu->hmr0.s.vmx.uTscDeadline += cTicksToDeadline;
4388 if (cTicksToDeadline >= 128)
4389 { /* hopefully */ }
4390 else
4391 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatVmxPreemptionRecalcingDeadlineExpired);
4392 }
4393
4394 /* Make sure the returned values have sane upper and lower boundaries. */
4395 uint64_t const u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
4396 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second, 15.625ms. */ /** @todo r=bird: Once real+virtual timers move to separate thread, we can raise the upper limit (16ms isn't much). ASSUMES working poke cpu function. */
4397 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 32678); /* 1/32768th of a second, ~30us. */
4398 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4399
4400 /** @todo r=ramshankar: We need to find a way to integrate nested-guest
4401 * preemption timers here. We probably need to clamp the preemption timer,
4402 * after converting the timer value to the host. */
4403 uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4404 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
4405 AssertRC(rc);
4406 }
4407 else
4408 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
4409
4410 if (fParavirtTsc)
4411 {
4412 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
4413 information before every VM-entry, hence disable it for performance sake. */
4414#if 0
4415 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
4416 AssertRC(rc);
4417#endif
4418 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
4419 }
4420
4421 if ( fOffsettedTsc
4422 && RT_LIKELY(!pVCpu->hmr0.s.fDebugWantRdTscExit))
4423 {
4424 if (pVmxTransient->fIsNestedGuest)
4425 uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
4426 hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
4427 hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
4428 }
4429 else
4430 {
4431 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4432 hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
4433 }
4434}
4435
4436
4437/**
4438 * Worker for VMXR0ImportStateOnDemand.
4439 *
4440 * @returns VBox status code.
4441 * @param pVCpu The cross context virtual CPU structure.
4442 * @param pVmcsInfo The VMCS info. object.
4443 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
4444 */
4445static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
4446{
4447 int rc = VINF_SUCCESS;
4448 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4449 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4450 uint32_t u32Val;
4451
4452 /*
4453 * Note! This is hack to workaround a mysterious BSOD observed with release builds
4454 * on Windows 10 64-bit hosts. Profile and debug builds are not affected and
4455 * neither are other host platforms.
4456 *
4457 * Committing this temporarily as it prevents BSOD.
4458 *
4459 * Update: This is very likely a compiler optimization bug, see @bugref{9180}.
4460 */
4461#ifdef RT_OS_WINDOWS
4462 if (pVM == 0 || pVM == (void *)(uintptr_t)-1)
4463 return VERR_HM_IPE_1;
4464#endif
4465
4466 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
4467
4468 /*
4469 * We disable interrupts to make the updating of the state and in particular
4470 * the fExtrn modification atomic wrt to preemption hooks.
4471 */
4472 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
4473
4474 fWhat &= pCtx->fExtrn;
4475 if (fWhat)
4476 {
4477 do
4478 {
4479 if (fWhat & CPUMCTX_EXTRN_RIP)
4480 vmxHCImportGuestRip(pVCpu);
4481
4482 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
4483 vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
4484
4485 if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
4486 vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
4487
4488 if (fWhat & CPUMCTX_EXTRN_RSP)
4489 {
4490 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
4491 AssertRC(rc);
4492 }
4493
4494 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
4495 {
4496 PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
4497 bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
4498 if (fWhat & CPUMCTX_EXTRN_CS)
4499 {
4500 vmxHCImportGuestSegReg(pVCpu, X86_SREG_CS);
4501 vmxHCImportGuestRip(pVCpu);
4502 if (fRealOnV86Active)
4503 pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
4504 EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
4505 }
4506 if (fWhat & CPUMCTX_EXTRN_SS)
4507 {
4508 vmxHCImportGuestSegReg(pVCpu, X86_SREG_SS);
4509 if (fRealOnV86Active)
4510 pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
4511 }
4512 if (fWhat & CPUMCTX_EXTRN_DS)
4513 {
4514 vmxHCImportGuestSegReg(pVCpu, X86_SREG_DS);
4515 if (fRealOnV86Active)
4516 pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
4517 }
4518 if (fWhat & CPUMCTX_EXTRN_ES)
4519 {
4520 vmxHCImportGuestSegReg(pVCpu, X86_SREG_ES);
4521 if (fRealOnV86Active)
4522 pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
4523 }
4524 if (fWhat & CPUMCTX_EXTRN_FS)
4525 {
4526 vmxHCImportGuestSegReg(pVCpu, X86_SREG_FS);
4527 if (fRealOnV86Active)
4528 pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
4529 }
4530 if (fWhat & CPUMCTX_EXTRN_GS)
4531 {
4532 vmxHCImportGuestSegReg(pVCpu, X86_SREG_GS);
4533 if (fRealOnV86Active)
4534 pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
4535 }
4536 }
4537
4538 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
4539 {
4540 if (fWhat & CPUMCTX_EXTRN_LDTR)
4541 vmxHCImportGuestLdtr(pVCpu);
4542
4543 if (fWhat & CPUMCTX_EXTRN_GDTR)
4544 {
4545 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
4546 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
4547 pCtx->gdtr.cbGdt = u32Val;
4548 }
4549
4550 /* Guest IDTR. */
4551 if (fWhat & CPUMCTX_EXTRN_IDTR)
4552 {
4553 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
4554 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
4555 pCtx->idtr.cbIdt = u32Val;
4556 }
4557
4558 /* Guest TR. */
4559 if (fWhat & CPUMCTX_EXTRN_TR)
4560 {
4561 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
4562 don't need to import that one. */
4563 if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
4564 vmxHCImportGuestTr(pVCpu);
4565 }
4566 }
4567
4568 if (fWhat & CPUMCTX_EXTRN_DR7)
4569 {
4570 if (!pVCpu->hmr0.s.fUsingHyperDR7)
4571 {
4572 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
4573 AssertRC(rc);
4574 }
4575 }
4576
4577 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
4578 {
4579 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
4580 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
4581 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
4582 pCtx->SysEnter.cs = u32Val;
4583 }
4584
4585 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
4586 {
4587 if ( pVM->hmr0.s.fAllow64BitGuests
4588 && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
4589 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
4590 }
4591
4592 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
4593 {
4594 if ( pVM->hmr0.s.fAllow64BitGuests
4595 && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
4596 {
4597 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
4598 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
4599 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
4600 }
4601 }
4602
4603 if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
4604 {
4605 PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
4606 PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
4607 uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
4608 Assert(pMsrs);
4609 Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
4610 Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
4611 for (uint32_t i = 0; i < cMsrs; i++)
4612 {
4613 uint32_t const idMsr = pMsrs[i].u32Msr;
4614 switch (idMsr)
4615 {
4616 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
4617 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
4618 case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
4619 default:
4620 {
4621 uint32_t idxLbrMsr;
4622 if (pVM->hmr0.s.vmx.fLbr)
4623 {
4624 if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
4625 {
4626 Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
4627 pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
4628 break;
4629 }
4630 if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
4631 {
4632 Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
4633 pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
4634 break;
4635 }
4636 if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
4637 {
4638 pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
4639 break;
4640 }
4641 /* Fallthru (no break) */
4642 }
4643 pCtx->fExtrn = 0;
4644 pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
4645 ASMSetFlags(fEFlags);
4646 AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
4647 return VERR_HM_UNEXPECTED_LD_ST_MSR;
4648 }
4649 }
4650 }
4651 }
4652
4653 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
4654 {
4655 if (fWhat & CPUMCTX_EXTRN_CR0)
4656 {
4657 uint64_t u64Cr0;
4658 uint64_t u64Shadow;
4659 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
4660 rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
4661#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
4662 u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
4663 | (u64Shadow & pVmcsInfo->u64Cr0Mask);
4664#else
4665 if (!CPUMIsGuestInVmxNonRootMode(pCtx))
4666 {
4667 u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
4668 | (u64Shadow & pVmcsInfo->u64Cr0Mask);
4669 }
4670 else
4671 {
4672 /*
4673 * We've merged the guest and nested-guest's CR0 guest/host mask while executing
4674 * the nested-guest using hardware-assisted VMX. Accordingly we need to
4675 * re-construct CR0. See @bugref{9180#c95} for details.
4676 */
4677 PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
4678 PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
4679 u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
4680 | (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
4681 | (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
4682 }
4683#endif
4684 VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
4685 CPUMSetGuestCR0(pVCpu, u64Cr0);
4686 VMMRZCallRing3Enable(pVCpu);
4687 }
4688
4689 if (fWhat & CPUMCTX_EXTRN_CR4)
4690 {
4691 uint64_t u64Cr4;
4692 uint64_t u64Shadow;
4693 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
4694 rc |= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
4695#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
4696 u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
4697 | (u64Shadow & pVmcsInfo->u64Cr4Mask);
4698#else
4699 if (!CPUMIsGuestInVmxNonRootMode(pCtx))
4700 {
4701 u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
4702 | (u64Shadow & pVmcsInfo->u64Cr4Mask);
4703 }
4704 else
4705 {
4706 /*
4707 * We've merged the guest and nested-guest's CR4 guest/host mask while executing
4708 * the nested-guest using hardware-assisted VMX. Accordingly we need to
4709 * re-construct CR4. See @bugref{9180#c95} for details.
4710 */
4711 PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
4712 PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
4713 u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
4714 | (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
4715 | (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
4716 }
4717#endif
4718 pCtx->cr4 = u64Cr4;
4719 }
4720
4721 if (fWhat & CPUMCTX_EXTRN_CR3)
4722 {
4723 /* CR0.PG bit changes are always intercepted, so it's up to date. */
4724 if ( pVM->hmr0.s.vmx.fUnrestrictedGuest
4725 || ( pVM->hmr0.s.fNestedPaging
4726 && CPUMIsGuestPagingEnabledEx(pCtx)))
4727 {
4728 uint64_t u64Cr3;
4729 rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
4730 if (pCtx->cr3 != u64Cr3)
4731 {
4732 pCtx->cr3 = u64Cr3;
4733 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
4734 }
4735
4736 /*
4737 * If the guest is in PAE mode, sync back the PDPE's into the guest state.
4738 * CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
4739 */
4740 if (CPUMIsGuestInPAEModeEx(pCtx))
4741 {
4742 X86PDPE aPaePdpes[4];
4743 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc);
4744 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc);
4745 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc);
4746 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc);
4747 if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
4748 {
4749 memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
4750 /* PGM now updates PAE PDPTEs while updating CR3. */
4751 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
4752 }
4753 }
4754 }
4755 }
4756 }
4757
4758#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4759 if (fWhat & CPUMCTX_EXTRN_HWVIRT)
4760 {
4761 if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
4762 && !CPUMIsGuestInVmxNonRootMode(pCtx))
4763 {
4764 Assert(CPUMIsGuestInVmxRootMode(pCtx));
4765 rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
4766 if (RT_SUCCESS(rc))
4767 { /* likely */ }
4768 else
4769 break;
4770 }
4771 }
4772#endif
4773 } while (0);
4774
4775 if (RT_SUCCESS(rc))
4776 {
4777 /* Update fExtrn. */
4778 pCtx->fExtrn &= ~fWhat;
4779
4780 /* If everything has been imported, clear the HM keeper bit. */
4781 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
4782 {
4783 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
4784 Assert(!pCtx->fExtrn);
4785 }
4786 }
4787 }
4788 else
4789 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
4790
4791 /*
4792 * Restore interrupts.
4793 */
4794 ASMSetFlags(fEFlags);
4795
4796 STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
4797
4798 if (RT_SUCCESS(rc))
4799 { /* likely */ }
4800 else
4801 return rc;
4802
4803 /*
4804 * Honor any pending CR3 updates.
4805 *
4806 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
4807 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
4808 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
4809 *
4810 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
4811 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
4812 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
4813 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
4814 *
4815 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
4816 *
4817 * The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
4818 */
4819 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
4820 && VMMRZCallRing3IsEnabled(pVCpu))
4821 {
4822 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
4823 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
4824 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
4825 }
4826
4827 return VINF_SUCCESS;
4828}
4829
4830
4831/**
4832 * Saves the guest state from the VMCS into the guest-CPU context.
4833 *
4834 * @returns VBox status code.
4835 * @param pVCpu The cross context virtual CPU structure.
4836 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
4837 */
4838VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
4839{
4840 AssertPtr(pVCpu);
4841 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4842 return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
4843}
4844
4845
4846/**
4847 * Gets VMX VM-exit auxiliary information.
4848 *
4849 * @returns VBox status code.
4850 * @param pVCpu The cross context virtual CPU structure.
4851 * @param pVmxExitAux Where to store the VM-exit auxiliary info.
4852 * @param fWhat What to fetch, HMVMX_READ_XXX.
4853 */
4854VMMR0DECL(int) VMXR0GetExitAuxInfo(PVMCPUCC pVCpu, PVMXEXITAUX pVmxExitAux, uint32_t fWhat)
4855{
4856 PVMXTRANSIENT pVmxTransient = pVCpu->hmr0.s.vmx.pVmxTransient;
4857 if (RT_LIKELY(pVmxTransient))
4858 {
4859 AssertCompile(sizeof(fWhat) == sizeof(pVmxTransient->fVmcsFieldsRead));
4860
4861 /* The exit reason is always available. */
4862 pVmxExitAux->uReason = pVmxTransient->uExitReason;
4863
4864 if (fWhat & HMVMX_READ_EXIT_QUALIFICATION)
4865 {
4866 vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
4867 pVmxExitAux->u64Qual = pVmxTransient->uExitQual;
4868#ifdef VBOX_STRICT
4869 fWhat &= ~HMVMX_READ_EXIT_QUALIFICATION;
4870#endif
4871 }
4872
4873 if (fWhat & HMVMX_READ_IDT_VECTORING_INFO)
4874 {
4875 vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
4876 pVmxExitAux->uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
4877#ifdef VBOX_STRICT
4878 fWhat &= ~HMVMX_READ_IDT_VECTORING_INFO;
4879#endif
4880 }
4881
4882 if (fWhat & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
4883 {
4884 vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
4885 pVmxExitAux->uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
4886#ifdef VBOX_STRICT
4887 fWhat &= ~HMVMX_READ_IDT_VECTORING_ERROR_CODE;
4888#endif
4889 }
4890
4891 if (fWhat & HMVMX_READ_EXIT_INSTR_LEN)
4892 {
4893 vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
4894 pVmxExitAux->cbInstr = pVmxTransient->cbExitInstr;
4895#ifdef VBOX_STRICT
4896 fWhat &= ~HMVMX_READ_EXIT_INSTR_LEN;
4897#endif
4898 }
4899
4900 if (fWhat & HMVMX_READ_EXIT_INTERRUPTION_INFO)
4901 {
4902 vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
4903 pVmxExitAux->uExitIntInfo = pVmxTransient->uExitIntInfo;
4904#ifdef VBOX_STRICT
4905 fWhat &= ~HMVMX_READ_EXIT_INTERRUPTION_INFO;
4906#endif
4907 }
4908
4909 if (fWhat & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
4910 {
4911 vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
4912 pVmxExitAux->uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
4913#ifdef VBOX_STRICT
4914 fWhat &= ~HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
4915#endif
4916 }
4917
4918 if (fWhat & HMVMX_READ_EXIT_INSTR_INFO)
4919 {
4920 vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
4921 pVmxExitAux->InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
4922#ifdef VBOX_STRICT
4923 fWhat &= ~HMVMX_READ_EXIT_INSTR_INFO;
4924#endif
4925 }
4926
4927 if (fWhat & HMVMX_READ_GUEST_LINEAR_ADDR)
4928 {
4929 vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
4930 pVmxExitAux->u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
4931#ifdef VBOX_STRICT
4932 fWhat &= ~HMVMX_READ_GUEST_LINEAR_ADDR;
4933#endif
4934 }
4935
4936 if (fWhat & HMVMX_READ_GUEST_PHYSICAL_ADDR)
4937 {
4938 vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
4939 pVmxExitAux->u64GuestPhysAddr = pVmxTransient->uGuestPhysicalAddr;
4940#ifdef VBOX_STRICT
4941 fWhat &= ~HMVMX_READ_GUEST_PHYSICAL_ADDR;
4942#endif
4943 }
4944
4945 if (fWhat & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
4946 {
4947#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4948 vmxHCReadGuestPendingDbgXctps(pVCpu, pVmxTransient);
4949 pVmxExitAux->u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
4950#else
4951 pVmxExitAux->u64GuestPendingDbgXcpts = 0;
4952#endif
4953#ifdef VBOX_STRICT
4954 fWhat &= ~HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
4955#endif
4956 }
4957
4958 AssertMsg(!fWhat, ("fWhat=%#RX32 fVmcsFieldsRead=%#RX32\n", fWhat, pVmxTransient->fVmcsFieldsRead));
4959 return VINF_SUCCESS;
4960 }
4961 return VERR_NOT_AVAILABLE;
4962}
4963
4964
4965/**
4966 * Does the necessary state syncing before returning to ring-3 for any reason
4967 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
4968 *
4969 * @returns VBox status code.
4970 * @param pVCpu The cross context virtual CPU structure.
4971 * @param fImportState Whether to import the guest state from the VMCS back
4972 * to the guest-CPU context.
4973 *
4974 * @remarks No-long-jmp zone!!!
4975 */
4976static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
4977{
4978 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4979 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4980
4981 RTCPUID const idCpu = RTMpCpuId();
4982 Log4Func(("HostCpuId=%u\n", idCpu));
4983
4984 /*
4985 * !!! IMPORTANT !!!
4986 * If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
4987 */
4988
4989 /* Save the guest state if necessary. */
4990 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4991 if (fImportState)
4992 {
4993 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
4994 AssertRCReturn(rc, rc);
4995 }
4996
4997 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
4998 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
4999 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5000
5001 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
5002#ifdef VBOX_STRICT
5003 if (CPUMIsHyperDebugStateActive(pVCpu))
5004 Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
5005#endif
5006 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
5007 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5008 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5009
5010 /* Restore host-state bits that VT-x only restores partially. */
5011 if (pVCpu->hmr0.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
5012 {
5013 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hmr0.s.vmx.fRestoreHostFlags, idCpu));
5014 VMXRestoreHostState(pVCpu->hmr0.s.vmx.fRestoreHostFlags, &pVCpu->hmr0.s.vmx.RestoreHost);
5015 }
5016 pVCpu->hmr0.s.vmx.fRestoreHostFlags = 0;
5017
5018 /* Restore the lazy host MSRs as we're leaving VT-x context. */
5019 if (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
5020 {
5021 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
5022 if (!fImportState)
5023 {
5024 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
5025 AssertRCReturn(rc, rc);
5026 }
5027 hmR0VmxLazyRestoreHostMsrs(pVCpu);
5028 Assert(!pVCpu->hmr0.s.vmx.fLazyMsrs);
5029 }
5030 else
5031 pVCpu->hmr0.s.vmx.fLazyMsrs = 0;
5032
5033 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
5034 pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
5035
5036 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
5037 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
5038 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
5039 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
5040 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
5041 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
5042 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
5043 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
5044 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
5045 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5046
5047 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5048
5049 /** @todo This partially defeats the purpose of having preemption hooks.
5050 * The problem is, deregistering the hooks should be moved to a place that
5051 * lasts until the EMT is about to be destroyed not everytime while leaving HM
5052 * context.
5053 */
5054 int rc = hmR0VmxClearVmcs(pVmcsInfo);
5055 AssertRCReturn(rc, rc);
5056
5057#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5058 /*
5059 * A valid shadow VMCS is made active as part of VM-entry. It is necessary to
5060 * clear a shadow VMCS before allowing that VMCS to become active on another
5061 * logical processor. We may or may not be importing guest state which clears
5062 * it, so cover for it here.
5063 *
5064 * See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
5065 */
5066 if ( pVmcsInfo->pvShadowVmcs
5067 && pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
5068 {
5069 rc = vmxHCClearShadowVmcs(pVmcsInfo);
5070 AssertRCReturn(rc, rc);
5071 }
5072
5073 /*
5074 * Flag that we need to re-export the host state if we switch to this VMCS before
5075 * executing guest or nested-guest code.
5076 */
5077 pVmcsInfo->idHostCpuState = NIL_RTCPUID;
5078#endif
5079
5080 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
5081 NOREF(idCpu);
5082 return VINF_SUCCESS;
5083}
5084
5085
5086/**
5087 * Leaves the VT-x session.
5088 *
5089 * @returns VBox status code.
5090 * @param pVCpu The cross context virtual CPU structure.
5091 *
5092 * @remarks No-long-jmp zone!!!
5093 */
5094static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
5095{
5096 HM_DISABLE_PREEMPT(pVCpu);
5097 HMVMX_ASSERT_CPU_SAFE(pVCpu);
5098 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5100
5101 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
5102 and done this from the VMXR0ThreadCtxCallback(). */
5103 if (!pVCpu->hmr0.s.fLeaveDone)
5104 {
5105 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
5106 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
5107 pVCpu->hmr0.s.fLeaveDone = true;
5108 }
5109 Assert(!pVCpu->cpum.GstCtx.fExtrn);
5110
5111 /*
5112 * !!! IMPORTANT !!!
5113 * If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
5114 */
5115
5116 /* Deregister hook now that we've left HM context before re-enabling preemption. */
5117 /** @todo Deregistering here means we need to VMCLEAR always
5118 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
5119 * for calling VMMR0ThreadCtxHookDisable here! */
5120 VMMR0ThreadCtxHookDisable(pVCpu);
5121
5122 /* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
5123 int rc = HMR0LeaveCpu(pVCpu);
5124 HM_RESTORE_PREEMPT();
5125 return rc;
5126}
5127
5128
5129/**
5130 * Take necessary actions before going back to ring-3.
5131 *
5132 * An action requires us to go back to ring-3. This function does the necessary
5133 * steps before we can safely return to ring-3. This is not the same as longjmps
5134 * to ring-3, this is voluntary and prepares the guest so it may continue
5135 * executing outside HM (recompiler/IEM).
5136 *
5137 * @returns VBox status code.
5138 * @param pVCpu The cross context virtual CPU structure.
5139 * @param rcExit The reason for exiting to ring-3. Can be
5140 * VINF_VMM_UNKNOWN_RING3_CALL.
5141 */
5142static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
5143{
5144 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5145
5146 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
5147 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5148 {
5149 VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
5150 pVCpu->hm.s.vmx.LastError.u32VmcsRev = *(uint32_t *)pVmcsInfo->pvVmcs;
5151 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hmr0.s.idEnteredCpu;
5152 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
5153 }
5154
5155 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5156 VMMRZCallRing3Disable(pVCpu);
5157 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
5158
5159 /*
5160 * Convert any pending HM events back to TRPM due to premature exits to ring-3.
5161 * We need to do this only on returns to ring-3 and not for longjmps to ring3.
5162 *
5163 * This is because execution may continue from ring-3 and we would need to inject
5164 * the event from there (hence place it back in TRPM).
5165 */
5166 if (pVCpu->hm.s.Event.fPending)
5167 {
5168 vmxHCPendingEventToTrpmTrap(pVCpu);
5169 Assert(!pVCpu->hm.s.Event.fPending);
5170
5171 /* Clear the events from the VMCS. */
5172 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
5173 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
5174 }
5175#ifdef VBOX_STRICT
5176 /*
5177 * We check for rcExit here since for errors like VERR_VMX_UNABLE_TO_START_VM (which are
5178 * fatal), we don't care about verifying duplicate injection of events. Errors like
5179 * VERR_EM_INTERPRET are converted to their VINF_* counterparts -prior- to calling this
5180 * function so those should and will be checked below.
5181 */
5182 else if (RT_SUCCESS(rcExit))
5183 {
5184 /*
5185 * Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
5186 * This can be pretty hard to debug otherwise, interrupts might get injected twice
5187 * occasionally, see @bugref{9180#c42}.
5188 *
5189 * However, if the VM-entry failed, any VM entry-interruption info. field would
5190 * be left unmodified as the event would not have been injected to the guest. In
5191 * such cases, don't assert, we're not going to continue guest execution anyway.
5192 */
5193 uint32_t uExitReason;
5194 uint32_t uEntryIntInfo;
5195 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
5196 rc |= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
5197 AssertRC(rc);
5198 AssertMsg(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) || !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo),
5199 ("uExitReason=%#RX32 uEntryIntInfo=%#RX32 rcExit=%d\n", uExitReason, uEntryIntInfo, VBOXSTRICTRC_VAL(rcExit)));
5200 }
5201#endif
5202
5203 /*
5204 * Clear the interrupt-window and NMI-window VMCS controls as we could have got
5205 * a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
5206 * (e.g. TPR below threshold).
5207 */
5208 if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
5209 {
5210 vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
5211 vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
5212 }
5213
5214 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
5215 and if we're injecting an event we should have a TRPM trap pending. */
5216 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
5217#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
5218 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
5219#endif
5220
5221 /* Save guest state and restore host state bits. */
5222 int rc = hmR0VmxLeaveSession(pVCpu);
5223 AssertRCReturn(rc, rc);
5224 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5225
5226 /* Thread-context hooks are unregistered at this point!!! */
5227 /* Ring-3 callback notifications are unregistered at this point!!! */
5228
5229 /* Sync recompiler state. */
5230 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5231 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5232 | CPUM_CHANGED_LDTR
5233 | CPUM_CHANGED_GDTR
5234 | CPUM_CHANGED_IDTR
5235 | CPUM_CHANGED_TR
5236 | CPUM_CHANGED_HIDDEN_SEL_REGS);
5237 if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
5238 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
5239 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5240
5241 Assert(!pVCpu->hmr0.s.fClearTrapFlag);
5242
5243 /* Update the exit-to-ring 3 reason. */
5244 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
5245
5246 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
5247 if ( rcExit != VINF_EM_RAW_INTERRUPT
5248 || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
5249 {
5250 Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
5251 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
5252 }
5253
5254 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5255 VMMRZCallRing3Enable(pVCpu);
5256 return rc;
5257}
5258
5259
5260/**
5261 * VMMRZCallRing3() callback wrapper which saves the guest state before we
5262 * longjump due to a ring-0 assertion.
5263 *
5264 * @returns VBox status code.
5265 * @param pVCpu The cross context virtual CPU structure.
5266 */
5267VMMR0DECL(int) VMXR0AssertionCallback(PVMCPUCC pVCpu)
5268{
5269 /*
5270 * !!! IMPORTANT !!!
5271 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
5272 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
5273 */
5274 VMMR0AssertionRemoveNotification(pVCpu);
5275 VMMRZCallRing3Disable(pVCpu);
5276 HM_DISABLE_PREEMPT(pVCpu);
5277
5278 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
5279 vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
5280 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
5281 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
5282
5283 /* Restore host-state bits that VT-x only restores partially. */
5284 if (pVCpu->hmr0.s.vmx.fRestoreHostFlags > VMX_RESTORE_HOST_REQUIRED)
5285 VMXRestoreHostState(pVCpu->hmr0.s.vmx.fRestoreHostFlags, &pVCpu->hmr0.s.vmx.RestoreHost);
5286 pVCpu->hmr0.s.vmx.fRestoreHostFlags = 0;
5287
5288 /* Restore the lazy host MSRs as we're leaving VT-x context. */
5289 if (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
5290 hmR0VmxLazyRestoreHostMsrs(pVCpu);
5291
5292 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
5293 pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
5294 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5295
5296 /* Clear the current VMCS data back to memory (shadow VMCS if any would have been
5297 cleared as part of importing the guest state above. */
5298 hmR0VmxClearVmcs(pVmcsInfo);
5299
5300 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
5301 VMMR0ThreadCtxHookDisable(pVCpu);
5302
5303 /* Leave HM context. This takes care of local init (term). */
5304 HMR0LeaveCpu(pVCpu);
5305 HM_RESTORE_PREEMPT();
5306 return VINF_SUCCESS;
5307}
5308
5309
5310/**
5311 * Enters the VT-x session.
5312 *
5313 * @returns VBox status code.
5314 * @param pVCpu The cross context virtual CPU structure.
5315 */
5316VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
5317{
5318 AssertPtr(pVCpu);
5319 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
5320 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5321
5322 LogFlowFunc(("pVCpu=%p\n", pVCpu));
5323 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
5324 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
5325
5326#ifdef VBOX_STRICT
5327 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
5328 RTCCUINTREG uHostCr4 = ASMGetCR4();
5329 if (!(uHostCr4 & X86_CR4_VMXE))
5330 {
5331 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
5332 return VERR_VMX_X86_CR4_VMXE_CLEARED;
5333 }
5334#endif
5335
5336 /*
5337 * Do the EMT scheduled L1D and MDS flush here if needed.
5338 */
5339 if (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_L1D_SCHED)
5340 ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
5341 else if (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_MDS_SCHED)
5342 hmR0MdsClear();
5343
5344 /*
5345 * Load the appropriate VMCS as the current and active one.
5346 */
5347 PVMXVMCSINFO pVmcsInfo;
5348 bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
5349 if (!fInNestedGuestMode)
5350 pVmcsInfo = &pVCpu->hmr0.s.vmx.VmcsInfo;
5351 else
5352 pVmcsInfo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
5353 int rc = hmR0VmxLoadVmcs(pVmcsInfo);
5354 if (RT_SUCCESS(rc))
5355 {
5356 pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
5357 pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fInNestedGuestMode;
5358 pVCpu->hmr0.s.fLeaveDone = false;
5359 Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
5360 }
5361 return rc;
5362}
5363
5364
5365/**
5366 * The thread-context callback.
5367 *
5368 * This is used together with RTThreadCtxHookCreate() on platforms which
5369 * supports it, and directly from VMMR0EmtPrepareForBlocking() and
5370 * VMMR0EmtResumeAfterBlocking() on platforms which don't.
5371 *
5372 * @param enmEvent The thread-context event.
5373 * @param pVCpu The cross context virtual CPU structure.
5374 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
5375 * @thread EMT(pVCpu)
5376 */
5377VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
5378{
5379 AssertPtr(pVCpu);
5380 RT_NOREF1(fGlobalInit);
5381
5382 switch (enmEvent)
5383 {
5384 case RTTHREADCTXEVENT_OUT:
5385 {
5386 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5387 VMCPU_ASSERT_EMT(pVCpu);
5388
5389 /* No longjmps (logger flushes, locks) in this fragile context. */
5390 VMMRZCallRing3Disable(pVCpu);
5391 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
5392
5393 /* Restore host-state (FPU, debug etc.) */
5394 if (!pVCpu->hmr0.s.fLeaveDone)
5395 {
5396 /*
5397 * Do -not- import the guest-state here as we might already be in the middle of importing
5398 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
5399 */
5400 hmR0VmxLeave(pVCpu, false /* fImportState */);
5401 pVCpu->hmr0.s.fLeaveDone = true;
5402 }
5403
5404 /* Leave HM context, takes care of local init (term). */
5405 int rc = HMR0LeaveCpu(pVCpu);
5406 AssertRC(rc);
5407
5408 /* Restore longjmp state. */
5409 VMMRZCallRing3Enable(pVCpu);
5410 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
5411 break;
5412 }
5413
5414 case RTTHREADCTXEVENT_IN:
5415 {
5416 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5417 VMCPU_ASSERT_EMT(pVCpu);
5418
5419 /* Do the EMT scheduled L1D and MDS flush here if needed. */
5420 if (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_L1D_SCHED)
5421 ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
5422 else if (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_MDS_SCHED)
5423 hmR0MdsClear();
5424
5425 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
5426 VMMRZCallRing3Disable(pVCpu);
5427 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
5428
5429 /* Initialize the bare minimum state required for HM. This takes care of
5430 initializing VT-x if necessary (onlined CPUs, local init etc.) */
5431 int rc = hmR0EnterCpu(pVCpu);
5432 AssertRC(rc);
5433 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
5434 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
5435
5436 /* Load the active VMCS as the current one. */
5437 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
5438 rc = hmR0VmxLoadVmcs(pVmcsInfo);
5439 AssertRC(rc);
5440 Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
5441 pVCpu->hmr0.s.fLeaveDone = false;
5442
5443 /* Restore longjmp state. */
5444 VMMRZCallRing3Enable(pVCpu);
5445 break;
5446 }
5447
5448 default:
5449 break;
5450 }
5451}
5452
5453
5454/**
5455 * Exports the host state into the VMCS host-state area.
5456 * Sets up the VM-exit MSR-load area.
5457 *
5458 * The CPU state will be loaded from these fields on every successful VM-exit.
5459 *
5460 * @returns VBox status code.
5461 * @param pVCpu The cross context virtual CPU structure.
5462 *
5463 * @remarks No-long-jump zone!!!
5464 */
5465static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
5466{
5467 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5468
5469 int rc = VINF_SUCCESS;
5470 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
5471 {
5472 uint64_t uHostCr4 = hmR0VmxExportHostControlRegs();
5473
5474 rc = hmR0VmxExportHostSegmentRegs(pVCpu, uHostCr4);
5475 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
5476
5477 hmR0VmxExportHostMsrs(pVCpu);
5478
5479 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
5480 }
5481 return rc;
5482}
5483
5484
5485/**
5486 * Saves the host state in the VMCS host-state.
5487 *
5488 * @returns VBox status code.
5489 * @param pVCpu The cross context virtual CPU structure.
5490 *
5491 * @remarks No-long-jump zone!!!
5492 */
5493VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
5494{
5495 AssertPtr(pVCpu);
5496 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5497
5498 /*
5499 * Export the host state here while entering HM context.
5500 * When thread-context hooks are used, we might get preempted and have to re-save the host
5501 * state but most of the time we won't be, so do it here before we disable interrupts.
5502 */
5503 return hmR0VmxExportHostState(pVCpu);
5504}
5505
5506
5507/**
5508 * Exports the guest state into the VMCS guest-state area.
5509 *
5510 * The will typically be done before VM-entry when the guest-CPU state and the
5511 * VMCS state may potentially be out of sync.
5512 *
5513 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
5514 * VM-entry controls.
5515 * Sets up the appropriate VMX non-root function to execute guest code based on
5516 * the guest CPU mode.
5517 *
5518 * @returns VBox strict status code.
5519 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5520 * without unrestricted guest execution and the VMMDev is not presently
5521 * mapped (e.g. EFI32).
5522 *
5523 * @param pVCpu The cross context virtual CPU structure.
5524 * @param pVmxTransient The VMX-transient structure.
5525 *
5526 * @remarks No-long-jump zone!!!
5527 */
5528static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5529{
5530 AssertPtr(pVCpu);
5531 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5532 LogFlowFunc(("pVCpu=%p\n", pVCpu));
5533
5534 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
5535
5536 /*
5537 * Determine real-on-v86 mode.
5538 * Used when the guest is in real-mode and unrestricted guest execution is not used.
5539 */
5540 PVMXVMCSINFOSHARED pVmcsInfoShared = pVmxTransient->pVmcsInfo->pShared;
5541 if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest
5542 || !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
5543 pVmcsInfoShared->RealMode.fRealOnV86Active = false;
5544 else
5545 {
5546 Assert(!pVmxTransient->fIsNestedGuest);
5547 pVmcsInfoShared->RealMode.fRealOnV86Active = true;
5548 }
5549
5550 /*
5551 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
5552 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
5553 */
5554 int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
5555 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
5556
5557 rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
5558 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
5559
5560 VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
5561 if (rcStrict == VINF_SUCCESS)
5562 { /* likely */ }
5563 else
5564 {
5565 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
5566 return rcStrict;
5567 }
5568
5569 rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
5570 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
5571
5572 rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
5573 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
5574
5575 vmxHCExportGuestApicTpr(pVCpu, pVmxTransient);
5576 vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
5577 vmxHCExportGuestRip(pVCpu);
5578 hmR0VmxExportGuestRsp(pVCpu);
5579 vmxHCExportGuestRflags(pVCpu, pVmxTransient);
5580
5581 rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
5582 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
5583
5584 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
5585 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
5586 | HM_CHANGED_GUEST_CR2
5587 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
5588 | HM_CHANGED_GUEST_X87
5589 | HM_CHANGED_GUEST_SSE_AVX
5590 | HM_CHANGED_GUEST_OTHER_XSAVE
5591 | HM_CHANGED_GUEST_XCRx
5592 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
5593 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
5594 | HM_CHANGED_GUEST_TSC_AUX
5595 | HM_CHANGED_GUEST_OTHER_MSRS
5596 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
5597
5598 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
5599 return rc;
5600}
5601
5602
5603/**
5604 * Exports the state shared between the host and guest into the VMCS.
5605 *
5606 * @param pVCpu The cross context virtual CPU structure.
5607 * @param pVmxTransient The VMX-transient structure.
5608 *
5609 * @remarks No-long-jump zone!!!
5610 */
5611static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5612{
5613 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5614 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5615
5616 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
5617 {
5618 int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
5619 AssertRC(rc);
5620 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
5621
5622 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
5623 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
5624 vmxHCExportGuestRflags(pVCpu, pVmxTransient);
5625 }
5626
5627 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
5628 {
5629 hmR0VmxLazyLoadGuestMsrs(pVCpu);
5630 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
5631 }
5632
5633 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
5634 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
5635}
5636
5637
5638/**
5639 * Worker for loading the guest-state bits in the inner VT-x execution loop.
5640 *
5641 * @returns Strict VBox status code (i.e. informational status codes too).
5642 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5643 * without unrestricted guest execution and the VMMDev is not presently
5644 * mapped (e.g. EFI32).
5645 *
5646 * @param pVCpu The cross context virtual CPU structure.
5647 * @param pVmxTransient The VMX-transient structure.
5648 *
5649 * @remarks No-long-jump zone!!!
5650 */
5651static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5652{
5653 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5654 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5655
5656#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
5657 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
5658#endif
5659
5660 /*
5661 * For many VM-exits only RIP/RSP/RFLAGS (and HWVIRT state when executing a nested-guest)
5662 * changes. First try to export only these without going through all other changed-flag checks.
5663 */
5664 VBOXSTRICTRC rcStrict;
5665 uint64_t const fCtxMask = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
5666 uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT;
5667 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5668
5669 /* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
5670 if ( (fCtxChanged & fMinimalMask)
5671 && !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
5672 {
5673 vmxHCExportGuestRip(pVCpu);
5674 hmR0VmxExportGuestRsp(pVCpu);
5675 vmxHCExportGuestRflags(pVCpu, pVmxTransient);
5676 rcStrict = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
5677 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
5678 }
5679 /* If anything else also changed, go through the full export routine and export as required. */
5680 else if (fCtxChanged & fCtxMask)
5681 {
5682 rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
5683 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5684 { /* likely */}
5685 else
5686 {
5687 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
5688 VBOXSTRICTRC_VAL(rcStrict)));
5689 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5690 return rcStrict;
5691 }
5692 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
5693 }
5694 /* Nothing changed, nothing to load here. */
5695 else
5696 rcStrict = VINF_SUCCESS;
5697
5698#ifdef VBOX_STRICT
5699 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
5700 uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5701 AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
5702#endif
5703 return rcStrict;
5704}
5705
5706
5707/**
5708 * Map the APIC-access page for virtualizing APIC accesses.
5709 *
5710 * This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
5711 * this not done as part of exporting guest state, see @bugref{8721}.
5712 *
5713 * @returns VBox status code.
5714 * @param pVCpu The cross context virtual CPU structure.
5715 * @param GCPhysApicBase The guest-physical address of the APIC access page.
5716 */
5717static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu, RTGCPHYS GCPhysApicBase)
5718{
5719 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5720 Assert(GCPhysApicBase);
5721
5722 LogFunc(("Mapping HC APIC-access page at %#RGp\n", GCPhysApicBase));
5723
5724 /* Unalias the existing mapping. */
5725 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
5726 AssertRCReturn(rc, rc);
5727
5728 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
5729 Assert(pVM->hmr0.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
5730 rc = IOMR0MmioMapMmioHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hmr0.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
5731 AssertRCReturn(rc, rc);
5732
5733 return VINF_SUCCESS;
5734}
5735
5736
5737/**
5738 * Worker function passed to RTMpOnSpecific() that is to be called on the target
5739 * CPU.
5740 *
5741 * @param idCpu The ID for the CPU the function is called on.
5742 * @param pvUser1 Null, not used.
5743 * @param pvUser2 Null, not used.
5744 */
5745static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void *pvUser1, void *pvUser2)
5746{
5747 RT_NOREF3(idCpu, pvUser1, pvUser2);
5748 VMXDispatchHostNmi();
5749}
5750
5751
5752/**
5753 * Dispatching an NMI on the host CPU that received it.
5754 *
5755 * @returns VBox status code.
5756 * @param pVCpu The cross context virtual CPU structure.
5757 * @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
5758 * executing when receiving the host NMI in VMX non-root
5759 * operation.
5760 */
5761static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
5762{
5763 RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
5764 Assert(idCpu != NIL_RTCPUID);
5765
5766 /*
5767 * We don't want to delay dispatching the NMI any more than we have to. However,
5768 * we have already chosen -not- to dispatch NMIs when interrupts were still disabled
5769 * after executing guest or nested-guest code for the following reasons:
5770 *
5771 * - We would need to perform VMREADs with interrupts disabled and is orders of
5772 * magnitude worse when we run as a nested hypervisor without VMCS shadowing
5773 * supported by the host hypervisor.
5774 *
5775 * - It affects the common VM-exit scenario and keeps interrupts disabled for a
5776 * longer period of time just for handling an edge case like host NMIs which do
5777 * not occur nearly as frequently as other VM-exits.
5778 *
5779 * Let's cover the most likely scenario first. Check if we are on the target CPU
5780 * and dispatch the NMI right away. This should be much faster than calling into
5781 * RTMpOnSpecific() machinery.
5782 */
5783 bool fDispatched = false;
5784 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5785 if (idCpu == RTMpCpuId())
5786 {
5787 VMXDispatchHostNmi();
5788 fDispatched = true;
5789 }
5790 ASMSetFlags(fEFlags);
5791 if (fDispatched)
5792 {
5793 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
5794 return VINF_SUCCESS;
5795 }
5796
5797 /*
5798 * RTMpOnSpecific() waits until the worker function has run on the target CPU. So
5799 * there should be no race or recursion even if we are unlucky enough to be preempted
5800 * (to the target CPU) without dispatching the host NMI above.
5801 */
5802 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
5803 return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 */, NULL /* pvUser2 */);
5804}
5805
5806
5807#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5808/**
5809 * Merges the guest with the nested-guest MSR bitmap in preparation of executing the
5810 * nested-guest using hardware-assisted VMX.
5811 *
5812 * @param pVCpu The cross context virtual CPU structure.
5813 * @param pVmcsInfoNstGst The nested-guest VMCS info. object.
5814 * @param pVmcsInfoGst The guest VMCS info. object.
5815 */
5816static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
5817{
5818 uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
5819 uint64_t *pu64MsrBitmap = (uint64_t *)pVmcsInfoNstGst->pvMsrBitmap;
5820 Assert(pu64MsrBitmap);
5821
5822 /*
5823 * We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
5824 * MSR that is intercepted by the guest is also intercepted while executing the
5825 * nested-guest using hardware-assisted VMX.
5826 *
5827 * Note! If the nested-guest is not using an MSR bitmap, every MSR must cause a
5828 * nested-guest VM-exit even if the outer guest is not intercepting some
5829 * MSRs. We cannot assume the caller has initialized the nested-guest
5830 * MSR bitmap in this case.
5831 *
5832 * The nested hypervisor may also switch whether it uses MSR bitmaps for
5833 * each of its VM-entry, hence initializing it once per-VM while setting
5834 * up the nested-guest VMCS is not sufficient.
5835 */
5836 PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
5837 if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
5838 {
5839 uint64_t const *pu64MsrBitmapNstGst = (uint64_t const *)&pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap[0];
5840 uint64_t const *pu64MsrBitmapGst = (uint64_t const *)pVmcsInfoGst->pvMsrBitmap;
5841 Assert(pu64MsrBitmapNstGst);
5842 Assert(pu64MsrBitmapGst);
5843
5844 /** @todo Detect and use EVEX.POR? */
5845 uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
5846 for (uint32_t i = 0; i < cFrags; i++)
5847 pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] | pu64MsrBitmapGst[i];
5848 }
5849 else
5850 ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
5851}
5852
5853
5854/**
5855 * Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
5856 * hardware-assisted VMX execution of the nested-guest.
5857 *
5858 * For a guest, we don't modify these controls once we set up the VMCS and hence
5859 * this function is never called.
5860 *
5861 * For nested-guests since the nested hypervisor provides these controls on every
5862 * nested-guest VM-entry and could potentially change them everytime we need to
5863 * merge them before every nested-guest VM-entry.
5864 *
5865 * @returns VBox status code.
5866 * @param pVCpu The cross context virtual CPU structure.
5867 */
5868static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
5869{
5870 PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
5871 PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
5872 PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
5873
5874 /*
5875 * Merge the controls with the requirements of the guest VMCS.
5876 *
5877 * We do not need to validate the nested-guest VMX features specified in the nested-guest
5878 * VMCS with the features supported by the physical CPU as it's already done by the
5879 * VMLAUNCH/VMRESUME instruction emulation.
5880 *
5881 * This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
5882 * derived from the VMX features supported by the physical CPU.
5883 */
5884
5885 /* Pin-based VM-execution controls. */
5886 uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls | pVmcsInfoGst->u32PinCtls;
5887
5888 /* Processor-based VM-execution controls. */
5889 uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
5890 | (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
5891 | VMX_PROC_CTLS_NMI_WINDOW_EXIT
5892 | VMX_PROC_CTLS_MOV_DR_EXIT
5893 | VMX_PROC_CTLS_USE_TPR_SHADOW
5894 | VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
5895
5896 /* Secondary processor-based VM-execution controls. */
5897 uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
5898 | (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
5899 | VMX_PROC_CTLS2_INVPCID
5900 | VMX_PROC_CTLS2_VMCS_SHADOWING
5901 | VMX_PROC_CTLS2_RDTSCP
5902 | VMX_PROC_CTLS2_XSAVES_XRSTORS
5903 | VMX_PROC_CTLS2_APIC_REG_VIRT
5904 | VMX_PROC_CTLS2_VIRT_INT_DELIVERY
5905 | VMX_PROC_CTLS2_VMFUNC));
5906
5907 /*
5908 * VM-entry controls:
5909 * These controls contains state that depends on the nested-guest state (primarily
5910 * EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
5911 * VM-exit. Although the nested hypervisor cannot change it, we need to in order to
5912 * properly continue executing the nested-guest if the EFER MSR changes but does not
5913 * cause a nested-guest VM-exits.
5914 *
5915 * VM-exit controls:
5916 * These controls specify the host state on return. We cannot use the controls from
5917 * the nested hypervisor state as is as it would contain the guest state rather than
5918 * the host state. Since the host state is subject to change (e.g. preemption, trips
5919 * to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
5920 * through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
5921 *
5922 * VM-entry MSR-load:
5923 * The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
5924 * context by the VMLAUNCH/VMRESUME instruction emulation.
5925 *
5926 * VM-exit MSR-store:
5927 * The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
5928 * back into the VM-exit MSR-store area.
5929 *
5930 * VM-exit MSR-load areas:
5931 * This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
5932 * can entirely ignore what the nested hypervisor wants to load here.
5933 */
5934
5935 /*
5936 * Exception bitmap.
5937 *
5938 * We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
5939 * here (and avoid doing anything while exporting nested-guest state), but to keep the
5940 * code more flexible if intercepting exceptions become more dynamic in the future we do
5941 * it as part of exporting the nested-guest state.
5942 */
5943 uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap | pVmcsInfoGst->u32XcptBitmap;
5944
5945 /*
5946 * CR0/CR4 guest/host mask.
5947 *
5948 * Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
5949 * cause VM-exits, so we need to merge them here.
5950 */
5951 uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u | pVmcsInfoGst->u64Cr0Mask;
5952 uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u | pVmcsInfoGst->u64Cr4Mask;
5953
5954 /*
5955 * Page-fault error-code mask and match.
5956 *
5957 * Although we require unrestricted guest execution (and thereby nested-paging) for
5958 * hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
5959 * normally intercept #PFs, it might intercept them for debugging purposes.
5960 *
5961 * If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
5962 * If the outer guest is intercepting #PFs, we must intercept all #PFs.
5963 */
5964 uint32_t u32XcptPFMask;
5965 uint32_t u32XcptPFMatch;
5966 if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
5967 {
5968 u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
5969 u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
5970 }
5971 else
5972 {
5973 u32XcptPFMask = 0;
5974 u32XcptPFMatch = 0;
5975 }
5976
5977 /*
5978 * Pause-Loop exiting.
5979 */
5980 /** @todo r=bird: given that both pVM->hm.s.vmx.cPleGapTicks and
5981 * pVM->hm.s.vmx.cPleWindowTicks defaults to zero, I cannot see how
5982 * this will work... */
5983 uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
5984 uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
5985
5986 /*
5987 * Pending debug exceptions.
5988 * Currently just copy whatever the nested-guest provides us.
5989 */
5990 uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
5991
5992 /*
5993 * I/O Bitmap.
5994 *
5995 * We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
5996 * intercept all I/O port accesses.
5997 */
5998 Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
5999 Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
6000
6001 /*
6002 * VMCS shadowing.
6003 *
6004 * We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
6005 * enabled while executing the nested-guest.
6006 */
6007 Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
6008
6009 /*
6010 * APIC-access page.
6011 */
6012 RTHCPHYS HCPhysApicAccess;
6013 if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
6014 {
6015 Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
6016 RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
6017
6018 /* Disabling this since we adjust the shadow page table permissions in (SyncHandlerPte). */
6019#if 0
6020 /** @todo NSTVMX: This is not really correct but currently is required to make
6021 * things work. We need to re-enable the page handler when we fallback to
6022 * IEM execution of the nested-guest! */
6023 PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
6024#endif
6025
6026 void *pvPage;
6027 PGMPAGEMAPLOCK PgLockApicAccess;
6028 int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
6029 if (RT_SUCCESS(rc))
6030 {
6031 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
6032 AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
6033
6034 /** @todo Handle proper releasing of page-mapping lock later. */
6035 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
6036 }
6037 else
6038 return rc;
6039 }
6040 else
6041 HCPhysApicAccess = 0;
6042
6043 /*
6044 * Virtual-APIC page and TPR threshold.
6045 */
6046 RTHCPHYS HCPhysVirtApic;
6047 uint32_t u32TprThreshold;
6048 if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
6049 {
6050 Assert(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
6051 RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
6052
6053 void *pvPage;
6054 PGMPAGEMAPLOCK PgLockVirtApic;
6055 int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
6056 if (RT_SUCCESS(rc))
6057 {
6058 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
6059 AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
6060
6061 /** @todo Handle proper releasing of page-mapping lock later. */
6062 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
6063 }
6064 else
6065 return rc;
6066
6067 u32TprThreshold = pVmcsNstGst->u32TprThreshold;
6068 }
6069 else
6070 {
6071 HCPhysVirtApic = 0;
6072 u32TprThreshold = 0;
6073
6074 /*
6075 * We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
6076 * used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
6077 * be taken care of by EPT/shadow paging.
6078 */
6079 if (pVM->hmr0.s.fAllow64BitGuests)
6080 u32ProcCtls |= VMX_PROC_CTLS_CR8_STORE_EXIT
6081 | VMX_PROC_CTLS_CR8_LOAD_EXIT;
6082 }
6083
6084 /*
6085 * Validate basic assumptions.
6086 */
6087 PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
6088 Assert(pVM->hmr0.s.vmx.fUnrestrictedGuest);
6089 Assert(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6090 Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
6091
6092 /*
6093 * Commit it to the nested-guest VMCS.
6094 */
6095 int rc = VINF_SUCCESS;
6096 if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
6097 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
6098 if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
6099 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
6100 if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
6101 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
6102 if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
6103 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
6104 if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
6105 rc |= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
6106 if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
6107 rc |= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
6108 if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
6109 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
6110 if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
6111 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
6112 if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
6113 && (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
6114 {
6115 Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
6116 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
6117 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
6118 }
6119 if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
6120 {
6121 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
6122 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
6123 }
6124 if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
6125 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
6126 rc |= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
6127 AssertRC(rc);
6128
6129 /*
6130 * Update the nested-guest VMCS cache.
6131 */
6132 pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
6133 pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
6134 pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
6135 pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
6136 pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
6137 pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
6138 pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
6139 pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
6140 pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
6141
6142 /*
6143 * We need to flush the TLB if we are switching the APIC-access page address.
6144 * See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
6145 */
6146 if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
6147 pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
6148
6149 /*
6150 * MSR bitmap.
6151 *
6152 * The MSR bitmap address has already been initialized while setting up the nested-guest
6153 * VMCS, here we need to merge the MSR bitmaps.
6154 */
6155 if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
6156 hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
6157
6158 return VINF_SUCCESS;
6159}
6160#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6161
6162
6163/**
6164 * Does the preparations before executing guest code in VT-x.
6165 *
6166 * This may cause longjmps to ring-3 and may even result in rescheduling to the
6167 * recompiler/IEM. We must be cautious what we do here regarding committing
6168 * guest-state information into the VMCS assuming we assuredly execute the
6169 * guest in VT-x mode.
6170 *
6171 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
6172 * the common-state (TRPM/forceflags), we must undo those changes so that the
6173 * recompiler/IEM can (and should) use them when it resumes guest execution.
6174 * Otherwise such operations must be done when we can no longer exit to ring-3.
6175 *
6176 * @returns Strict VBox status code (i.e. informational status codes too).
6177 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
6178 * have been disabled.
6179 * @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
6180 * pending events).
6181 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
6182 * double-fault into the guest.
6183 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
6184 * dispatched directly.
6185 * @retval VINF_* scheduling changes, we have to go back to ring-3.
6186 *
6187 * @param pVCpu The cross context virtual CPU structure.
6188 * @param pVmxTransient The VMX-transient structure.
6189 * @param fStepping Whether we are single-stepping the guest in the
6190 * hypervisor debugger. Makes us ignore some of the reasons
6191 * for returning to ring-3, and return VINF_EM_DBG_STEPPED
6192 * if event dispatching took place.
6193 */
6194static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
6195{
6196 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6197
6198 Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
6199
6200#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
6201 if (pVmxTransient->fIsNestedGuest)
6202 {
6203 RT_NOREF2(pVCpu, fStepping);
6204 Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
6205 return VINF_EM_RESCHEDULE_REM;
6206 }
6207#endif
6208
6209 /*
6210 * Check and process force flag actions, some of which might require us to go back to ring-3.
6211 */
6212 VBOXSTRICTRC rcStrict = vmxHCCheckForceFlags(pVCpu, pVmxTransient->fIsNestedGuest, fStepping);
6213 if (rcStrict == VINF_SUCCESS)
6214 {
6215 /* FFs don't get set all the time. */
6216#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6217 if ( pVmxTransient->fIsNestedGuest
6218 && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6219 {
6220 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
6221 return VINF_VMX_VMEXIT;
6222 }
6223#endif
6224 }
6225 else
6226 return rcStrict;
6227
6228 /*
6229 * Virtualize memory-mapped accesses to the physical APIC (may take locks).
6230 */
6231 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6232 if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
6233 && (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
6234 && PDMHasApic(pVM))
6235 {
6236 /* Get the APIC base MSR from the virtual APIC device. */
6237 uint64_t const uApicBaseMsr = APICGetBaseMsrNoCheck(pVCpu);
6238
6239 /* Map the APIC access page. */
6240 int rc = hmR0VmxMapHCApicAccessPage(pVCpu, uApicBaseMsr & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
6241 AssertRCReturn(rc, rc);
6242
6243 /* Update the per-VCPU cache of the APIC base MSR corresponding to the mapped APIC access page. */
6244 pVCpu->hm.s.vmx.u64GstMsrApicBase = uApicBaseMsr;
6245 }
6246
6247#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6248 /*
6249 * Merge guest VMCS controls with the nested-guest VMCS controls.
6250 *
6251 * Even if we have not executed the guest prior to this (e.g. when resuming from a
6252 * saved state), we should be okay with merging controls as we initialize the
6253 * guest VMCS controls as part of VM setup phase.
6254 */
6255 if ( pVmxTransient->fIsNestedGuest
6256 && !pVCpu->hm.s.vmx.fMergedNstGstCtls)
6257 {
6258 int rc = hmR0VmxMergeVmcsNested(pVCpu);
6259 AssertRCReturn(rc, rc);
6260 pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
6261 }
6262#endif
6263
6264 /*
6265 * Evaluate events to be injected into the guest.
6266 *
6267 * Events in TRPM can be injected without inspecting the guest state.
6268 * If any new events (interrupts/NMI) are pending currently, we try to set up the
6269 * guest to cause a VM-exit the next time they are ready to receive the event.
6270 */
6271 if (TRPMHasTrap(pVCpu))
6272 vmxHCTrpmTrapToPendingEvent(pVCpu);
6273
6274 uint32_t fIntrState;
6275 rcStrict = vmxHCEvaluatePendingEvent(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->fIsNestedGuest,
6276 &fIntrState);
6277
6278#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6279 /*
6280 * While evaluating pending events if something failed (unlikely) or if we were
6281 * preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
6282 */
6283 if (rcStrict != VINF_SUCCESS)
6284 return rcStrict;
6285 if ( pVmxTransient->fIsNestedGuest
6286 && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6287 {
6288 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
6289 return VINF_VMX_VMEXIT;
6290 }
6291#else
6292 Assert(rcStrict == VINF_SUCCESS);
6293#endif
6294
6295 /*
6296 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
6297 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
6298 * also result in triple-faulting the VM.
6299 *
6300 * With nested-guests, the above does not apply since unrestricted guest execution is a
6301 * requirement. Regardless, we do this here to avoid duplicating code elsewhere.
6302 */
6303 rcStrict = vmxHCInjectPendingEvent(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->fIsNestedGuest,
6304 fIntrState, fStepping);
6305 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
6306 { /* likely */ }
6307 else
6308 {
6309 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
6310 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6311 return rcStrict;
6312 }
6313
6314 /*
6315 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
6316 * import CR3 themselves. We will need to update them here, as even as late as the above
6317 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
6318 * the below force flags to be set.
6319 */
6320 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6321 {
6322 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
6323 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6324 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6325 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6326 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6327 }
6328
6329#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6330 /* Paranoia. */
6331 Assert(!pVmxTransient->fIsNestedGuest || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
6332#endif
6333
6334 /*
6335 * No longjmps to ring-3 from this point on!!!
6336 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6337 * This also disables flushing of the R0-logger instance (if any).
6338 */
6339 VMMRZCallRing3Disable(pVCpu);
6340
6341 /*
6342 * Export the guest state bits.
6343 *
6344 * We cannot perform longjmps while loading the guest state because we do not preserve the
6345 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
6346 * CPU migration.
6347 *
6348 * If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
6349 * registers. Hence, exporting of the guest state needs to be done -after- injection of events.
6350 */
6351 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
6352 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
6353 { /* likely */ }
6354 else
6355 {
6356 VMMRZCallRing3Enable(pVCpu);
6357 return rcStrict;
6358 }
6359
6360 /*
6361 * We disable interrupts so that we don't miss any interrupts that would flag preemption
6362 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
6363 * preemption disabled for a while. Since this is purely to aid the
6364 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
6365 * disable interrupt on NT.
6366 *
6367 * We need to check for force-flags that could've possible been altered since we last
6368 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
6369 * see @bugref{6398}).
6370 *
6371 * We also check a couple of other force-flags as a last opportunity to get the EMT back
6372 * to ring-3 before executing guest code.
6373 */
6374 pVmxTransient->fEFlags = ASMIntDisableFlags();
6375
6376 if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
6377 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6378 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
6379 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
6380 {
6381 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
6382 {
6383#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6384 /*
6385 * If we are executing a nested-guest make sure that we should intercept subsequent
6386 * events. The one we are injecting might be part of VM-entry. This is mainly to keep
6387 * the VM-exit instruction emulation happy.
6388 */
6389 if (pVmxTransient->fIsNestedGuest)
6390 CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
6391#endif
6392
6393 /*
6394 * We've injected any pending events. This is really the point of no return (to ring-3).
6395 *
6396 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
6397 * returns from this function, so do -not- enable them here.
6398 */
6399 pVCpu->hm.s.Event.fPending = false;
6400 return VINF_SUCCESS;
6401 }
6402
6403 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
6404 rcStrict = VINF_EM_RAW_INTERRUPT;
6405 }
6406 else
6407 {
6408 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6409 rcStrict = VINF_EM_RAW_TO_R3;
6410 }
6411
6412 ASMSetFlags(pVmxTransient->fEFlags);
6413 VMMRZCallRing3Enable(pVCpu);
6414
6415 return rcStrict;
6416}
6417
6418
6419/**
6420 * Final preparations before executing guest code using hardware-assisted VMX.
6421 *
6422 * We can no longer get preempted to a different host CPU and there are no returns
6423 * to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
6424 * failures), this function is not intended to fail sans unrecoverable hardware
6425 * errors.
6426 *
6427 * @param pVCpu The cross context virtual CPU structure.
6428 * @param pVmxTransient The VMX-transient structure.
6429 *
6430 * @remarks Called with preemption disabled.
6431 * @remarks No-long-jump zone!!!
6432 */
6433static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6434{
6435 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6437 Assert(!pVCpu->hm.s.Event.fPending);
6438
6439 /*
6440 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
6441 */
6442 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6443 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6444
6445 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6446 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6447 PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
6448 RTCPUID const idCurrentCpu = pHostCpu->idCpu;
6449
6450 if (!CPUMIsGuestFPUStateActive(pVCpu))
6451 {
6452 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
6453 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
6454 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
6455 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
6456 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
6457 }
6458
6459 /*
6460 * Re-export the host state bits as we may've been preempted (only happens when
6461 * thread-context hooks are used or when the VM start function changes) or if
6462 * the host CR0 is modified while loading the guest FPU state above.
6463 *
6464 * The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
6465 * changed the switcher back to 32-bit, we *must* save the 32-bit host state here,
6466 * see @bugref{8432}.
6467 *
6468 * This may also happen when switching to/from a nested-guest VMCS without leaving
6469 * ring-0.
6470 */
6471 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
6472 {
6473 hmR0VmxExportHostState(pVCpu);
6474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
6475 }
6476 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
6477
6478 /*
6479 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
6480 */
6481 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
6482 hmR0VmxExportSharedState(pVCpu, pVmxTransient);
6483 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
6484
6485 /*
6486 * Store status of the shared guest/host debug state at the time of VM-entry.
6487 */
6488 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
6489 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
6490
6491 /*
6492 * Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
6493 * more than one conditional check. The post-run side of our code shall determine
6494 * if it needs to sync. the virtual APIC TPR with the TPR-shadow.
6495 */
6496 if (pVmcsInfo->pbVirtApic)
6497 pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
6498
6499 /*
6500 * Update the host MSRs values in the VM-exit MSR-load area.
6501 */
6502 if (!pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs)
6503 {
6504 if (pVmcsInfo->cExitMsrLoad > 0)
6505 hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
6506 pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = true;
6507 }
6508
6509 /*
6510 * Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
6511 * VMX-preemption timer based on the next virtual sync clock deadline.
6512 */
6513 if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
6514 || idCurrentCpu != pVCpu->hmr0.s.idLastCpu)
6515 {
6516 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient, idCurrentCpu);
6517 pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
6518 }
6519
6520 /* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
6521 bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
6522 if (!fIsRdtscIntercepted)
6523 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6524 else
6525 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6526
6527 ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
6528 hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
6529 Assert(idCurrentCpu == pVCpu->hmr0.s.idLastCpu);
6530 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
6531 pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
6532 pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
6533
6534 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6535
6536 TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
6537 as we're about to start executing the guest. */
6538
6539 /*
6540 * Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
6541 *
6542 * This is done this late as updating the TSC offsetting/preemption timer above
6543 * figures out if we can skip intercepting RDTSCP by calculating the number of
6544 * host CPU ticks till the next virtual sync deadline (for the dynamic case).
6545 */
6546 if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
6547 && !fIsRdtscIntercepted)
6548 {
6549 vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
6550
6551 /* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
6552 it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
6553 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
6554 true /* fSetReadWrite */, true /* fUpdateHostMsr */);
6555 AssertRC(rc);
6556 Assert(!pVmxTransient->fRemoveTscAuxMsr);
6557 pVmxTransient->fRemoveTscAuxMsr = true;
6558 }
6559
6560#ifdef VBOX_STRICT
6561 Assert(pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs);
6562 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
6563 hmR0VmxCheckHostEferMsr(pVmcsInfo);
6564 AssertRC(vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
6565#endif
6566
6567#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
6568 /** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
6569 * Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
6570 * see @bugref{9180#c54}. */
6571 uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
6572 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
6573 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
6574#endif
6575}
6576
6577
6578/**
6579 * First C routine invoked after running guest code using hardware-assisted VMX.
6580 *
6581 * @param pVCpu The cross context virtual CPU structure.
6582 * @param pVmxTransient The VMX-transient structure.
6583 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6584 *
6585 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
6586 *
6587 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
6588 * unconditionally when it is safe to do so.
6589 */
6590static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6591{
6592 ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
6593 ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
6594 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6595 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6596 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6597 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
6598
6599 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6600 if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6601 {
6602 uint64_t uGstTsc;
6603 if (!pVmxTransient->fIsNestedGuest)
6604 uGstTsc = pVCpu->hmr0.s.uTscExit + pVmcsInfo->u64TscOffset;
6605 else
6606 {
6607 uint64_t const uNstGstTsc = pVCpu->hmr0.s.uTscExit + pVmcsInfo->u64TscOffset;
6608 uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
6609 }
6610 TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
6611 }
6612
6613 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
6614 TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
6615 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6616
6617 pVCpu->hmr0.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
6618 pVmcsInfo->fVmcsState |= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6619#ifdef VBOX_STRICT
6620 hmR0VmxCheckHostEferMsr(pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
6621#endif
6622 Assert(!ASMIntAreEnabled());
6623 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
6624 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6625
6626#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
6627 /*
6628 * Clean all the VMCS fields in the transient structure before reading
6629 * anything from the VMCS.
6630 */
6631 pVmxTransient->uExitReason = 0;
6632 pVmxTransient->uExitIntErrorCode = 0;
6633 pVmxTransient->uExitQual = 0;
6634 pVmxTransient->uGuestLinearAddr = 0;
6635 pVmxTransient->uExitIntInfo = 0;
6636 pVmxTransient->cbExitInstr = 0;
6637 pVmxTransient->ExitInstrInfo.u = 0;
6638 pVmxTransient->uEntryIntInfo = 0;
6639 pVmxTransient->uEntryXcptErrorCode = 0;
6640 pVmxTransient->cbEntryInstr = 0;
6641 pVmxTransient->uIdtVectoringInfo = 0;
6642 pVmxTransient->uIdtVectoringErrorCode = 0;
6643#endif
6644
6645 /*
6646 * Save the basic VM-exit reason and check if the VM-entry failed.
6647 * See Intel spec. 24.9.1 "Basic VM-exit Information".
6648 */
6649 uint32_t uExitReason;
6650 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6651 AssertRC(rc);
6652 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
6653 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
6654
6655 /*
6656 * Log the VM-exit before logging anything else as otherwise it might be a
6657 * tad confusing what happens before and after the world-switch.
6658 */
6659 HMVMX_LOG_EXIT(pVCpu, uExitReason);
6660
6661 /*
6662 * Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
6663 * bitmap permissions, if it was added before VM-entry.
6664 */
6665 if (pVmxTransient->fRemoveTscAuxMsr)
6666 {
6667 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
6668 pVmxTransient->fRemoveTscAuxMsr = false;
6669 }
6670
6671 /*
6672 * Check if VMLAUNCH/VMRESUME succeeded.
6673 * If this failed, we cause a guru meditation and cease further execution.
6674 */
6675 if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
6676 {
6677 /*
6678 * Update the VM-exit history array here even if the VM-entry failed due to:
6679 * - Invalid guest state.
6680 * - MSR loading.
6681 * - Machine-check event.
6682 *
6683 * In any of the above cases we will still have a "valid" VM-exit reason
6684 * despite @a fVMEntryFailed being false.
6685 *
6686 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
6687 *
6688 * Note! We don't have CS or RIP at this point. Will probably address that later
6689 * by amending the history entry added here.
6690 */
6691 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
6692 UINT64_MAX, pVCpu->hmr0.s.uTscExit);
6693
6694 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6695 {
6696 VMMRZCallRing3Enable(pVCpu);
6697 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6698
6699#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
6700 hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
6701#endif
6702
6703 /*
6704 * Always import the guest-interruptibility state as we need it while evaluating
6705 * injecting events on re-entry.
6706 *
6707 * We don't import CR0 (when unrestricted guest execution is unavailable) despite
6708 * checking for real-mode while exporting the state because all bits that cause
6709 * mode changes wrt CR0 are intercepted.
6710 */
6711 uint64_t const fImportMask = CPUMCTX_EXTRN_INHIBIT_INT
6712 | CPUMCTX_EXTRN_INHIBIT_NMI
6713#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
6714 | HMVMX_CPUMCTX_EXTRN_ALL
6715#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
6716 | CPUMCTX_EXTRN_RFLAGS
6717#endif
6718 ;
6719 rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImportMask);
6720 AssertRC(rc);
6721
6722 /*
6723 * Sync the TPR shadow with our APIC state.
6724 */
6725 if ( !pVmxTransient->fIsNestedGuest
6726 && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
6727 {
6728 Assert(pVmcsInfo->pbVirtApic);
6729 if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
6730 {
6731 rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
6732 AssertRC(rc);
6733 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6734 }
6735 }
6736
6737 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6738 Assert( pVmxTransient->fWasGuestDebugStateActive == false
6739 || pVmxTransient->fWasHyperDebugStateActive == false);
6740 return;
6741 }
6742 }
6743#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6744 else if (pVmxTransient->fIsNestedGuest)
6745 AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
6746#endif
6747 else
6748 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6749
6750 VMMRZCallRing3Enable(pVCpu);
6751}
6752
6753
6754/**
6755 * Runs the guest code using hardware-assisted VMX the normal way.
6756 *
6757 * @returns VBox status code.
6758 * @param pVCpu The cross context virtual CPU structure.
6759 * @param pcLoops Pointer to the number of executed loops.
6760 */
6761static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
6762{
6763 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6764 Assert(pcLoops);
6765 Assert(*pcLoops <= cMaxResumeLoops);
6766 Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
6767
6768#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6769 /*
6770 * Switch to the guest VMCS as we may have transitioned from executing the nested-guest
6771 * without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
6772 * guest VMCS while entering the VMX ring-0 session.
6773 */
6774 if (pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
6775 {
6776 int rc = vmxHCSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
6777 if (RT_SUCCESS(rc))
6778 { /* likely */ }
6779 else
6780 {
6781 LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
6782 return rc;
6783 }
6784 }
6785#endif
6786
6787 VMXTRANSIENT VmxTransient;
6788 RT_ZERO(VmxTransient);
6789 VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6790
6791 /* Paranoia. */
6792 Assert(VmxTransient.pVmcsInfo == &pVCpu->hmr0.s.vmx.VmcsInfo);
6793
6794 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
6795 for (;;)
6796 {
6797 Assert(!HMR0SuspendPending());
6798 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6799 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6800
6801 /*
6802 * Preparatory work for running nested-guest code, this may force us to
6803 * return to ring-3.
6804 *
6805 * Warning! This bugger disables interrupts on VINF_SUCCESS!
6806 */
6807 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
6808 if (rcStrict != VINF_SUCCESS)
6809 break;
6810
6811 /* Interrupts are disabled at this point! */
6812 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
6813 int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
6814 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
6815 /* Interrupts are re-enabled at this point! */
6816
6817 /*
6818 * Check for errors with running the VM (VMLAUNCH/VMRESUME).
6819 */
6820 if (RT_SUCCESS(rcRun))
6821 { /* very likely */ }
6822 else
6823 {
6824 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6825 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
6826 return rcRun;
6827 }
6828
6829 /*
6830 * Profile the VM-exit.
6831 */
6832 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6833 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
6834 STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6835 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6836 HMVMX_START_EXIT_DISPATCH_PROF();
6837
6838 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
6839
6840 /*
6841 * Handle the VM-exit.
6842 */
6843#ifdef HMVMX_USE_FUNCTION_TABLE
6844 rcStrict = g_aVMExitHandlers[VmxTransient.uExitReason].pfn(pVCpu, &VmxTransient);
6845#else
6846 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
6847#endif
6848 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6849 if (rcStrict == VINF_SUCCESS)
6850 {
6851 if (++(*pcLoops) <= cMaxResumeLoops)
6852 continue;
6853 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6854 rcStrict = VINF_EM_RAW_INTERRUPT;
6855 }
6856 break;
6857 }
6858
6859 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6860 return rcStrict;
6861}
6862
6863
6864#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6865/**
6866 * Runs the nested-guest code using hardware-assisted VMX.
6867 *
6868 * @returns VBox status code.
6869 * @param pVCpu The cross context virtual CPU structure.
6870 * @param pcLoops Pointer to the number of executed loops.
6871 *
6872 * @sa hmR0VmxRunGuestCodeNormal.
6873 */
6874static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
6875{
6876 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6877 Assert(pcLoops);
6878 Assert(*pcLoops <= cMaxResumeLoops);
6879 Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
6880
6881 /*
6882 * Switch to the nested-guest VMCS as we may have transitioned from executing the
6883 * guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
6884 * loaded the nested-guest VMCS while entering the VMX ring-0 session.
6885 */
6886 if (!pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
6887 {
6888 int rc = vmxHCSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
6889 if (RT_SUCCESS(rc))
6890 { /* likely */ }
6891 else
6892 {
6893 LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
6894 return rc;
6895 }
6896 }
6897
6898 VMXTRANSIENT VmxTransient;
6899 RT_ZERO(VmxTransient);
6900 VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6901 VmxTransient.fIsNestedGuest = true;
6902
6903 /* Paranoia. */
6904 Assert(VmxTransient.pVmcsInfo == &pVCpu->hmr0.s.vmx.VmcsInfoNstGst);
6905
6906 /* Setup pointer so PGM/IEM can query VM-exit auxiliary info. on demand in ring-0. */
6907 pVCpu->hmr0.s.vmx.pVmxTransient = &VmxTransient;
6908
6909 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
6910 for (;;)
6911 {
6912 Assert(!HMR0SuspendPending());
6913 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6914 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6915
6916 /*
6917 * Preparatory work for running guest code, this may force us to
6918 * return to ring-3.
6919 *
6920 * Warning! This bugger disables interrupts on VINF_SUCCESS!
6921 */
6922 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
6923 if (rcStrict != VINF_SUCCESS)
6924 break;
6925
6926 /* Interrupts are disabled at this point! */
6927 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
6928 int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
6929 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
6930 /* Interrupts are re-enabled at this point! */
6931
6932 /*
6933 * Check for errors with running the VM (VMLAUNCH/VMRESUME).
6934 */
6935 if (RT_SUCCESS(rcRun))
6936 { /* very likely */ }
6937 else
6938 {
6939 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6940 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
6941 rcStrict = rcRun;
6942 break;
6943 }
6944
6945 /* Disabling this since we adjust the shadow page table permissions in (SyncHandlerPte). */
6946#if 0
6947 /*
6948 * Undo temporary disabling of the APIC-access page monitoring we did in hmR0VmxMergeVmcsNested.
6949 * This is needed for NestedTrap0eHandler (and IEM) to cause nested-guest APIC-access VM-exits.
6950 */
6951 if (VmxTransient.pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
6952 {
6953 PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
6954 RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
6955 PGMHandlerPhysicalReset(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess);
6956 }
6957#endif
6958
6959 /*
6960 * Profile the VM-exit.
6961 */
6962 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6963 STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
6964 STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6965 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6966 HMVMX_START_EXIT_DISPATCH_PROF();
6967
6968 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
6969
6970 /*
6971 * Handle the VM-exit.
6972 */
6973 rcStrict = vmxHCHandleExitNested(pVCpu, &VmxTransient);
6974 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6975 if (rcStrict == VINF_SUCCESS)
6976 {
6977 if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6978 {
6979 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
6980 rcStrict = VINF_VMX_VMEXIT;
6981 }
6982 else
6983 {
6984 if (++(*pcLoops) <= cMaxResumeLoops)
6985 continue;
6986 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6987 rcStrict = VINF_EM_RAW_INTERRUPT;
6988 }
6989 }
6990 else
6991 Assert(rcStrict != VINF_VMX_VMEXIT);
6992 break;
6993 }
6994
6995 /* Ensure VM-exit auxiliary info. is no longer available. */
6996 pVCpu->hmr0.s.vmx.pVmxTransient = NULL;
6997
6998 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6999 return rcStrict;
7000}
7001#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7002
7003
7004/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
7005 * probes.
7006 *
7007 * The following few functions and associated structure contains the bloat
7008 * necessary for providing detailed debug events and dtrace probes as well as
7009 * reliable host side single stepping. This works on the principle of
7010 * "subclassing" the normal execution loop and workers. We replace the loop
7011 * method completely and override selected helpers to add necessary adjustments
7012 * to their core operation.
7013 *
7014 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
7015 * any performance for debug and analysis features.
7016 *
7017 * @{
7018 */
7019
7020/**
7021 * Single steps guest code using hardware-assisted VMX.
7022 *
7023 * This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
7024 * but single-stepping through the hypervisor debugger.
7025 *
7026 * @returns Strict VBox status code (i.e. informational status codes too).
7027 * @param pVCpu The cross context virtual CPU structure.
7028 * @param pcLoops Pointer to the number of executed loops.
7029 *
7030 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
7031 */
7032static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
7033{
7034 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
7035 Assert(pcLoops);
7036 Assert(*pcLoops <= cMaxResumeLoops);
7037
7038 VMXTRANSIENT VmxTransient;
7039 RT_ZERO(VmxTransient);
7040 VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7041
7042 /* Set HMCPU indicators. */
7043 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
7044 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
7045 pVCpu->hmr0.s.fDebugWantRdTscExit = false;
7046 pVCpu->hmr0.s.fUsingDebugLoop = true;
7047
7048 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
7049 VMXRUNDBGSTATE DbgState;
7050 vmxHCRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
7051 vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
7052
7053 /*
7054 * The loop.
7055 */
7056 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
7057 for (;;)
7058 {
7059 Assert(!HMR0SuspendPending());
7060 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7061 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7062 bool fStepping = pVCpu->hm.s.fSingleInstruction;
7063
7064 /* Set up VM-execution controls the next two can respond to. */
7065 vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
7066
7067 /*
7068 * Preparatory work for running guest code, this may force us to
7069 * return to ring-3.
7070 *
7071 * Warning! This bugger disables interrupts on VINF_SUCCESS!
7072 */
7073 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
7074 if (rcStrict != VINF_SUCCESS)
7075 break;
7076
7077 /* Interrupts are disabled at this point! */
7078 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
7079
7080 /* Override any obnoxious code in the above two calls. */
7081 vmxHCPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
7082
7083 /*
7084 * Finally execute the guest.
7085 */
7086 int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
7087
7088 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
7089 /* Interrupts are re-enabled at this point! */
7090
7091 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7092 if (RT_SUCCESS(rcRun))
7093 { /* very likely */ }
7094 else
7095 {
7096 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
7097 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
7098 return rcRun;
7099 }
7100
7101 /* Profile the VM-exit. */
7102 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7103 STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll);
7104 STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7105 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
7106 HMVMX_START_EXIT_DISPATCH_PROF();
7107
7108 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
7109
7110 /*
7111 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
7112 */
7113 rcStrict = vmxHCRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
7114 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
7115 if (rcStrict != VINF_SUCCESS)
7116 break;
7117 if (++(*pcLoops) > cMaxResumeLoops)
7118 {
7119 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
7120 rcStrict = VINF_EM_RAW_INTERRUPT;
7121 break;
7122 }
7123
7124 /*
7125 * Stepping: Did the RIP change, if so, consider it a single step.
7126 * Otherwise, make sure one of the TFs gets set.
7127 */
7128 if (fStepping)
7129 {
7130 int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
7131 AssertRC(rc);
7132 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
7133 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
7134 {
7135 rcStrict = VINF_EM_DBG_STEPPED;
7136 break;
7137 }
7138 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
7139 }
7140
7141 /*
7142 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
7143 */
7144 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
7145 vmxHCPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
7146
7147 /* Restore all controls applied by hmR0VmxPreRunGuestDebugStateApply above. */
7148 rcStrict = vmxHCRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
7149 Assert(rcStrict == VINF_SUCCESS);
7150 }
7151
7152 /*
7153 * Clear the X86_EFL_TF if necessary.
7154 */
7155 if (pVCpu->hmr0.s.fClearTrapFlag)
7156 {
7157 int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
7158 AssertRC(rc);
7159 pVCpu->hmr0.s.fClearTrapFlag = false;
7160 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
7161 }
7162 /** @todo there seems to be issues with the resume flag when the monitor trap
7163 * flag is pending without being used. Seen early in bios init when
7164 * accessing APIC page in protected mode. */
7165
7166/** @todo we need to do hmR0VmxRunDebugStateRevert here too, in case we broke
7167 * out of the above loop. */
7168
7169 /* Restore HMCPU indicators. */
7170 pVCpu->hmr0.s.fUsingDebugLoop = false;
7171 pVCpu->hmr0.s.fDebugWantRdTscExit = false;
7172 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
7173
7174 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7175 return rcStrict;
7176}
7177
7178/** @} */
7179
7180
7181/**
7182 * Checks if any expensive dtrace probes are enabled and we should go to the
7183 * debug loop.
7184 *
7185 * @returns true if we should use debug loop, false if not.
7186 */
7187static bool hmR0VmxAnyExpensiveProbesEnabled(void)
7188{
7189 /* It's probably faster to OR the raw 32-bit counter variables together.
7190 Since the variables are in an array and the probes are next to one
7191 another (more or less), we have good locality. So, better read
7192 eight-nine cache lines ever time and only have one conditional, than
7193 128+ conditionals, right? */
7194 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
7195 | VBOXVMM_XCPT_DE_ENABLED_RAW()
7196 | VBOXVMM_XCPT_DB_ENABLED_RAW()
7197 | VBOXVMM_XCPT_BP_ENABLED_RAW()
7198 | VBOXVMM_XCPT_OF_ENABLED_RAW()
7199 | VBOXVMM_XCPT_BR_ENABLED_RAW()
7200 | VBOXVMM_XCPT_UD_ENABLED_RAW()
7201 | VBOXVMM_XCPT_NM_ENABLED_RAW()
7202 | VBOXVMM_XCPT_DF_ENABLED_RAW()
7203 | VBOXVMM_XCPT_TS_ENABLED_RAW()
7204 | VBOXVMM_XCPT_NP_ENABLED_RAW()
7205 | VBOXVMM_XCPT_SS_ENABLED_RAW()
7206 | VBOXVMM_XCPT_GP_ENABLED_RAW()
7207 | VBOXVMM_XCPT_PF_ENABLED_RAW()
7208 | VBOXVMM_XCPT_MF_ENABLED_RAW()
7209 | VBOXVMM_XCPT_AC_ENABLED_RAW()
7210 | VBOXVMM_XCPT_XF_ENABLED_RAW()
7211 | VBOXVMM_XCPT_VE_ENABLED_RAW()
7212 | VBOXVMM_XCPT_SX_ENABLED_RAW()
7213 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
7214 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
7215 ) != 0
7216 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
7217 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
7218 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
7219 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
7220 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
7221 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
7222 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
7223 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
7224 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
7225 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
7226 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
7227 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
7228 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
7229 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
7230 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
7231 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
7232 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
7233 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
7234 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
7235 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
7236 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
7237 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
7238 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
7239 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
7240 | VBOXVMM_INSTR_STR_ENABLED_RAW()
7241 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
7242 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
7243 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
7244 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
7245 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
7246 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
7247 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
7248 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
7249 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
7250 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
7251 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
7252 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
7253 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
7254 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
7255 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
7256 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
7257 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
7258 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
7259 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
7260 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
7261 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
7262 ) != 0
7263 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
7264 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
7265 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
7266 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
7267 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
7268 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
7269 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
7270 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
7271 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
7272 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
7273 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
7274 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
7275 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
7276 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
7277 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
7278 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
7279 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
7280 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
7281 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
7282 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
7283 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
7284 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
7285 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
7286 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
7287 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
7288 | VBOXVMM_EXIT_STR_ENABLED_RAW()
7289 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
7290 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
7291 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
7292 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
7293 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
7294 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
7295 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
7296 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
7297 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
7298 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
7299 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
7300 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
7301 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
7302 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
7303 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
7304 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
7305 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
7306 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
7307 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
7308 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
7309 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
7310 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
7311 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
7312 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
7313 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
7314 ) != 0;
7315}
7316
7317
7318/**
7319 * Runs the guest using hardware-assisted VMX.
7320 *
7321 * @returns Strict VBox status code (i.e. informational status codes too).
7322 * @param pVCpu The cross context virtual CPU structure.
7323 */
7324VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
7325{
7326 AssertPtr(pVCpu);
7327 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7328 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7329 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
7330 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7331
7332 VBOXSTRICTRC rcStrict;
7333 uint32_t cLoops = 0;
7334 for (;;)
7335 {
7336#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7337 bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
7338#else
7339 NOREF(pCtx);
7340 bool const fInNestedGuestMode = false;
7341#endif
7342 if (!fInNestedGuestMode)
7343 {
7344 if ( !pVCpu->hm.s.fUseDebugLoop
7345 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
7346 && !DBGFIsStepping(pVCpu)
7347 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
7348 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
7349 else
7350 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
7351 }
7352#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7353 else
7354 rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
7355
7356 if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
7357 {
7358 Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
7359 continue;
7360 }
7361 if (rcStrict == VINF_VMX_VMEXIT)
7362 {
7363 Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
7364 continue;
7365 }
7366#endif
7367 break;
7368 }
7369
7370 int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
7371 switch (rcLoop)
7372 {
7373 case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
7374 case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
7375 }
7376
7377 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
7378 if (RT_FAILURE(rc2))
7379 {
7380 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
7381 rcStrict = rc2;
7382 }
7383 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
7384 Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
7385 return rcStrict;
7386}
7387
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette