/* $Id: PGMInline.h 99051 2023-03-19 16:40:06Z vboxsync $ */ /** @file * PGM - Inlined functions. */ /* * Copyright (C) 2006-2023 Oracle and/or its affiliates. * * This file is part of VirtualBox base platform packages, as * available from https://www.virtualbox.org. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation, in version 3 of the * License. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, see . * * SPDX-License-Identifier: GPL-3.0-only */ #ifndef VMM_INCLUDED_SRC_include_PGMInline_h #define VMM_INCLUDED_SRC_include_PGMInline_h #ifndef RT_WITHOUT_PRAGMA_ONCE # pragma once #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** @addtogroup grp_pgm_int Internals * @internal * @{ */ /** * Gets the PGMRAMRANGE structure for a guest page. * * @returns Pointer to the RAM range on success. * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition. * * @param pVM The cross context VM structure. * @param GCPhys The GC physical address. */ DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PVMCC pVM, RTGCPHYS GCPhys) { PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)]; if (!pRam || GCPhys - pRam->GCPhys >= pRam->cb) return pgmPhysGetRangeSlow(pVM, GCPhys); STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbHits)); return pRam; } /** * Gets the PGMRAMRANGE structure for a guest page, if unassigned get the ram * range above it. * * @returns Pointer to the RAM range on success. * @returns NULL if the address is located after the last range. * * @param pVM The cross context VM structure. * @param GCPhys The GC physical address. */ DECLINLINE(PPGMRAMRANGE) pgmPhysGetRangeAtOrAbove(PVMCC pVM, RTGCPHYS GCPhys) { PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)]; if ( !pRam || (GCPhys - pRam->GCPhys) >= pRam->cb) return pgmPhysGetRangeAtOrAboveSlow(pVM, GCPhys); STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbHits)); return pRam; } /** * Gets the PGMPAGE structure for a guest page. * * @returns Pointer to the page on success. * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition. * * @param pVM The cross context VM structure. * @param GCPhys The GC physical address. */ DECLINLINE(PPGMPAGE) pgmPhysGetPage(PVMCC pVM, RTGCPHYS GCPhys) { PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)]; RTGCPHYS off; if ( pRam && (off = GCPhys - pRam->GCPhys) < pRam->cb) { STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbHits)); return &pRam->aPages[off >> GUEST_PAGE_SHIFT]; } return pgmPhysGetPageSlow(pVM, GCPhys); } /** * Gets the PGMPAGE structure for a guest page. * * Old Phys code: Will make sure the page is present. * * @returns VBox status code. * @retval VINF_SUCCESS and a valid *ppPage on success. * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid. * * @param pVM The cross context VM structure. * @param GCPhys The GC physical address. * @param ppPage Where to store the page pointer on success. */ DECLINLINE(int) pgmPhysGetPageEx(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage) { PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)]; RTGCPHYS off; if ( !pRam || (off = GCPhys - pRam->GCPhys) >= pRam->cb) return pgmPhysGetPageExSlow(pVM, GCPhys, ppPage); *ppPage = &pRam->aPages[off >> GUEST_PAGE_SHIFT]; STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbHits)); return VINF_SUCCESS; } /** * Gets the PGMPAGE structure for a guest page. * * Old Phys code: Will make sure the page is present. * * @returns VBox status code. * @retval VINF_SUCCESS and a valid *ppPage on success. * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid. * * @param pVM The cross context VM structure. * @param GCPhys The GC physical address. * @param ppPage Where to store the page pointer on success. * @param ppRamHint Where to read and store the ram list hint. * The caller initializes this to NULL before the call. */ DECLINLINE(int) pgmPhysGetPageWithHintEx(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint) { RTGCPHYS off; PPGMRAMRANGE pRam = *ppRamHint; if ( !pRam || RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb)) { pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)]; if ( !pRam || (off = GCPhys - pRam->GCPhys) >= pRam->cb) return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRamHint); STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbHits)); *ppRamHint = pRam; } *ppPage = &pRam->aPages[off >> GUEST_PAGE_SHIFT]; return VINF_SUCCESS; } /** * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE. * * @returns Pointer to the page on success. * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition. * * @param pVM The cross context VM structure. * @param GCPhys The GC physical address. * @param ppPage Where to store the pointer to the PGMPAGE structure. * @param ppRam Where to store the pointer to the PGMRAMRANGE structure. */ DECLINLINE(int) pgmPhysGetPageAndRangeEx(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam) { PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)]; RTGCPHYS off; if ( !pRam || (off = GCPhys - pRam->GCPhys) >= pRam->cb) return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRam); STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbHits)); *ppRam = pRam; *ppPage = &pRam->aPages[off >> GUEST_PAGE_SHIFT]; return VINF_SUCCESS; } /** * Convert GC Phys to HC Phys. * * @returns VBox status code. * @param pVM The cross context VM structure. * @param GCPhys The GC physical address. * @param pHCPhys Where to store the corresponding HC physical address. * * @deprecated Doesn't deal with zero, shared or write monitored pages. * Avoid when writing new code! */ DECLINLINE(int) pgmRamGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys) { PPGMPAGE pPage; int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage); if (RT_FAILURE(rc)) return rc; *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & GUEST_PAGE_OFFSET_MASK); return VINF_SUCCESS; } /** * Queries the Physical TLB entry for a physical guest page, * attempting to load the TLB entry if necessary. * * @returns VBox status code. * @retval VINF_SUCCESS on success * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. * * @param pVM The cross context VM structure. * @param GCPhys The address of the guest page. * @param ppTlbe Where to store the pointer to the TLB entry. */ DECLINLINE(int) pgmPhysPageQueryTlbe(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe) { int rc; PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)]; if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK)) { STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbHits)); rc = VINF_SUCCESS; } else rc = pgmPhysPageLoadIntoTlb(pVM, GCPhys); *ppTlbe = pTlbe; return rc; } /** * Queries the Physical TLB entry for a physical guest page, * attempting to load the TLB entry if necessary. * * @returns VBox status code. * @retval VINF_SUCCESS on success * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. * * @param pVM The cross context VM structure. * @param pPage Pointer to the PGMPAGE structure corresponding to * GCPhys. * @param GCPhys The address of the guest page. * @param ppTlbe Where to store the pointer to the TLB entry. */ DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe) { int rc; PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)]; if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK)) { STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbHits)); rc = VINF_SUCCESS; AssertPtr(pTlbe->pv); #ifdef IN_RING3 Assert(!pTlbe->pMap || RT_VALID_PTR(pTlbe->pMap->pv)); #endif } else rc = pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys); *ppTlbe = pTlbe; return rc; } /** * Calculates NEM page protection flags. */ DECL_FORCE_INLINE(uint32_t) pgmPhysPageCalcNemProtection(PPGMPAGE pPage, PGMPAGETYPE enmType) { /* * Deal with potentially writable pages first. */ if (PGMPAGETYPE_IS_RWX(enmType)) { if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)) { if (PGM_PAGE_IS_ALLOCATED(pPage)) return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE | NEM_PAGE_PROT_WRITE; return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE; } if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE; } /* * Potentially readable & executable pages. */ else if ( PGMPAGETYPE_IS_ROX(enmType) && !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE; /* * The rest is needs special access handling. */ return NEM_PAGE_PROT_NONE; } /** * Enables write monitoring for an allocated page. * * The caller is responsible for updating the shadow page tables. * * @param pVM The cross context VM structure. * @param pPage The page to write monitor. * @param GCPhysPage The address of the page. */ DECLINLINE(void) pgmPhysPageWriteMonitor(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage) { Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); PGM_LOCK_ASSERT_OWNER(pVM); PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_WRITE_MONITORED); pVM->pgm.s.cMonitoredPages++; /* Large pages must disabled. */ if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE) { PPGMPAGE pFirstPage = pgmPhysGetPage(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK); AssertFatal(pFirstPage); if (PGM_PAGE_GET_PDE_TYPE(pFirstPage) == PGM_PAGE_PDE_TYPE_PDE) { PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED); pVM->pgm.s.cLargePagesDisabled++; } else Assert(PGM_PAGE_GET_PDE_TYPE(pFirstPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED); } #ifdef VBOX_WITH_NATIVE_NEM /* Tell NEM. */ if (VM_IS_NEM_ENABLED(pVM)) { uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhysPage); NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage) : NULL, pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State); PGM_PAGE_SET_NEM_STATE(pPage, u2State); } #endif } #ifndef VBOX_VMM_TARGET_ARMV8 /** * Checks if the no-execute (NX) feature is active (EFER.NXE=1). * * Only used when the guest is in PAE or long mode. This is inlined so that we * can perform consistency checks in debug builds. * * @returns true if it is, false if it isn't. * @param pVCpu The cross context virtual CPU structure. */ DECL_FORCE_INLINE(bool) pgmGstIsNoExecuteActive(PVMCPUCC pVCpu) { Assert(pVCpu->pgm.s.fNoExecuteEnabled == CPUMIsGuestNXEnabled(pVCpu)); Assert(CPUMIsGuestInPAEMode(pVCpu) || CPUMIsGuestInLongMode(pVCpu)); return pVCpu->pgm.s.fNoExecuteEnabled; } /** * Checks if the page size extension (PSE) is currently enabled (CR4.PSE=1). * * Only used when the guest is in paged 32-bit mode. This is inlined so that * we can perform consistency checks in debug builds. * * @returns true if it is, false if it isn't. * @param pVCpu The cross context virtual CPU structure. */ DECL_FORCE_INLINE(bool) pgmGst32BitIsPageSizeExtActive(PVMCPUCC pVCpu) { Assert(pVCpu->pgm.s.fGst32BitPageSizeExtension == CPUMIsGuestPageSizeExtEnabled(pVCpu)); Assert(!CPUMIsGuestInPAEMode(pVCpu)); Assert(!CPUMIsGuestInLongMode(pVCpu)); return pVCpu->pgm.s.fGst32BitPageSizeExtension; } /** * Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode. * Takes PSE-36 into account. * * @returns guest physical address * @param pVM The cross context VM structure. * @param Pde Guest Pde */ DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PVMCC pVM, X86PDE Pde) { RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK; GCPhys |= (RTGCPHYS)(Pde.u & X86_PDE4M_PG_HIGH_MASK) << X86_PDE4M_PG_HIGH_SHIFT; return GCPhys & pVM->pgm.s.GCPhys4MBPSEMask; } /** * Gets the address the guest page directory (32-bit paging). * * @returns VBox status code. * @param pVCpu The cross context virtual CPU structure. * @param ppPd Where to return the mapping. This is always set. */ DECLINLINE(int) pgmGstGet32bitPDPtrEx(PVMCPUCC pVCpu, PX86PD *ppPd) { *ppPd = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd); if (RT_UNLIKELY(!*ppPd)) return pgmGstLazyMap32BitPD(pVCpu, ppPd); return VINF_SUCCESS; } /** * Gets the address the guest page directory (32-bit paging). * * @returns Pointer to the page directory entry in question. * @param pVCpu The cross context virtual CPU structure. */ DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PVMCPUCC pVCpu) { PX86PD pGuestPD = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd); if (RT_UNLIKELY(!pGuestPD)) { int rc = pgmGstLazyMap32BitPD(pVCpu, &pGuestPD); if (RT_FAILURE(rc)) return NULL; } return pGuestPD; } /** * Gets the guest page directory pointer table. * * @returns VBox status code. * @param pVCpu The cross context virtual CPU structure. * @param ppPdpt Where to return the mapping. This is always set. */ DECLINLINE(int) pgmGstGetPaePDPTPtrEx(PVMCPUCC pVCpu, PX86PDPT *ppPdpt) { *ppPdpt = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt); if (RT_UNLIKELY(!*ppPdpt)) return pgmGstLazyMapPaePDPT(pVCpu, ppPdpt); return VINF_SUCCESS; } /** * Gets the guest page directory pointer table. * * @returns Pointer to the page directory in question. * @returns NULL if the page directory is not present or on an invalid page. * @param pVCpu The cross context virtual CPU structure. */ DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PVMCPUCC pVCpu) { PX86PDPT pGuestPdpt; int rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGuestPdpt); AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); return pGuestPdpt; } /** * Gets the guest page directory pointer table entry for the specified address. * * @returns Pointer to the page directory in question. * @returns NULL if the page directory is not present or on an invalid page. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. */ DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr) { AssertGCPtr32(GCPtr); PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt); if (RT_UNLIKELY(!pGuestPDPT)) { int rc = pgmGstLazyMapPaePDPT(pVCpu, &pGuestPDPT); if (RT_FAILURE(rc)) return NULL; } return &pGuestPDPT->a[(uint32_t)GCPtr >> X86_PDPT_SHIFT]; } /** * Gets the page directory entry for the specified address. * * @returns The page directory entry in question. * @returns A non-present entry if the page directory is not present or on an invalid page. * @param pVCpu The cross context virtual CPU structure of the calling EMT. * @param GCPtr The address. */ DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PVMCPUCC pVCpu, RTGCPTR GCPtr) { AssertGCPtr32(GCPtr); PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu); if (RT_LIKELY(pGuestPDPT)) { const unsigned iPdpt = (uint32_t)GCPtr >> X86_PDPT_SHIFT; if ((pGuestPDPT->a[iPdpt].u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask | X86_PDPE_P)) == X86_PDPE_P) { const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK; PX86PDPAE pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt]; if ( !pGuestPD || (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt]) pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD); if (pGuestPD) return pGuestPD->a[iPD]; } } X86PDEPAE ZeroPde = {0}; return ZeroPde; } /** * Gets the page directory pointer table entry for the specified address * and returns the index into the page directory * * @returns Pointer to the page directory in question. * @returns NULL if the page directory is not present or on an invalid page. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. * @param piPD Receives the index into the returned page directory * @param pPdpe Receives the page directory pointer entry. Optional. */ DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe) { AssertGCPtr32(GCPtr); /* The PDPE. */ PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu); if (pGuestPDPT) { const unsigned iPdpt = (uint32_t)GCPtr >> X86_PDPT_SHIFT; X86PGPAEUINT const uPdpe = pGuestPDPT->a[iPdpt].u; if (pPdpe) pPdpe->u = uPdpe; if ((uPdpe & (pVCpu->pgm.s.fGstPaeMbzPdpeMask | X86_PDPE_P)) == X86_PDPE_P) { /* The PDE. */ PX86PDPAE pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt]; if ( !pGuestPD || (uPdpe & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt]) pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD); *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK; return pGuestPD; } } return NULL; } /** * Gets the page map level-4 pointer for the guest. * * @returns VBox status code. * @param pVCpu The cross context virtual CPU structure. * @param ppPml4 Where to return the mapping. Always set. */ DECLINLINE(int) pgmGstGetLongModePML4PtrEx(PVMCPUCC pVCpu, PX86PML4 *ppPml4) { *ppPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4); if (RT_UNLIKELY(!*ppPml4)) return pgmGstLazyMapPml4(pVCpu, ppPml4); return VINF_SUCCESS; } /** * Gets the page map level-4 pointer for the guest. * * @returns Pointer to the PML4 page. * @param pVCpu The cross context virtual CPU structure. */ DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PVMCPUCC pVCpu) { PX86PML4 pGuestPml4; int rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGuestPml4); AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); return pGuestPml4; } /** * Gets the pointer to a page map level-4 entry. * * @returns Pointer to the PML4 entry. * @param pVCpu The cross context virtual CPU structure. * @param iPml4 The index. * @remarks Only used by AssertCR3. */ DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PVMCPUCC pVCpu, unsigned int iPml4) { PX86PML4 pGuestPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4); if (pGuestPml4) { /* likely */ } else { int rc = pgmGstLazyMapPml4(pVCpu, &pGuestPml4); AssertRCReturn(rc, NULL); } return &pGuestPml4->a[iPml4]; } /** * Gets the page directory entry for the specified address. * * @returns The page directory entry in question. * @returns A non-present entry if the page directory is not present or on an invalid page. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. */ DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PVMCPUCC pVCpu, RTGCPTR64 GCPtr) { /* * Note! To keep things simple, ASSUME invalid physical addresses will * cause X86_TRAP_PF_RSVD. This isn't a problem until we start * supporting 52-bit wide physical guest addresses. */ PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu); if (RT_LIKELY(pGuestPml4)) { const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK; X86PGPAEUINT const uPml4e = pGuestPml4->a[iPml4].u; if ((uPml4e & (pVCpu->pgm.s.fGstAmd64MbzPml4eMask | X86_PML4E_P)) == X86_PML4E_P) { PCX86PDPT pPdptTemp; int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, uPml4e & X86_PML4E_PG_MASK, &pPdptTemp); if (RT_SUCCESS(rc)) { const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; X86PGPAEUINT const uPdpte = pPdptTemp->a[iPdpt].u; if ((uPdpte & (pVCpu->pgm.s.fGstAmd64MbzPdpeMask | X86_PDPE_P)) == X86_PDPE_P) { PCX86PDPAE pPD; rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, uPdpte & X86_PDPE_PG_MASK, &pPD); if (RT_SUCCESS(rc)) { const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK; return pPD->a[iPD]; } } } AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); } } X86PDEPAE ZeroPde = {0}; return ZeroPde; } /** * Gets the GUEST page directory pointer for the specified address. * * @returns The page directory in question. * @returns NULL if the page directory is not present or on an invalid page. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. * @param ppPml4e Page Map Level-4 Entry (out) * @param pPdpe Page directory pointer table entry (out) * @param piPD Receives the index into the returned page directory */ DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe, unsigned *piPD) { /* The PMLE4. */ PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu); if (pGuestPml4) { const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK; *ppPml4e = &pGuestPml4->a[iPml4]; X86PGPAEUINT const uPml4e = pGuestPml4->a[iPml4].u; if ((uPml4e & (pVCpu->pgm.s.fGstAmd64MbzPml4eMask | X86_PML4E_P)) == X86_PML4E_P) { /* The PDPE. */ PCX86PDPT pPdptTemp; int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, uPml4e & X86_PML4E_PG_MASK, &pPdptTemp); if (RT_SUCCESS(rc)) { const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; X86PGPAEUINT const uPdpe = pPdptTemp->a[iPdpt].u; pPdpe->u = uPdpe; if ((uPdpe & (pVCpu->pgm.s.fGstAmd64MbzPdpeMask | X86_PDPE_P)) == X86_PDPE_P) { /* The PDE. */ PX86PDPAE pPD; rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, uPdpe & X86_PDPE_PG_MASK, &pPD); if (RT_SUCCESS(rc)) { *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK; return pPD; } AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); } } else AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); } } return NULL; } #ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT # if 0 /** * Gets the pointer to a page map level-4 entry when the guest using EPT paging. * * @returns Pointer to the PML4 entry. * @param pVCpu The cross context virtual CPU structure. * @param iPml4 The index. * @remarks Only used by AssertCR3. */ DECLINLINE(PEPTPML4E) pgmGstGetEptPML4EPtr(PVMCPUCC pVCpu, unsigned int iPml4) { PEPTPML4 pEptPml4 = pVCpu->pgm.s.CTX_SUFF(pGstEptPml4); if (pEptPml4) { /* likely */ } else { int const rc = pgmGstLazyMapEptPml4(pVCpu, &pEptPml4); AssertRCReturn(rc, NULL); } return &pEptPml4->a[iPml4]; } # endif /** * Gets the page map level-4 pointer for the guest when the guest is using EPT * paging. * * @returns VBox status code. * @param pVCpu The cross context virtual CPU structure. * @param ppEptPml4 Where to return the mapping. Always set. */ DECLINLINE(int) pgmGstGetEptPML4PtrEx(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4) { /* Shadow CR3 might not have been mapped at this point, see PGMHCChangeMode. */ *ppEptPml4 = pVCpu->pgm.s.CTX_SUFF(pGstEptPml4); if (!*ppEptPml4) return pgmGstLazyMapEptPml4(pVCpu, ppEptPml4); return VINF_SUCCESS; } # if 0 /** * Gets the page map level-4 pointer for the guest when the guest is using EPT * paging. * * @returns Pointer to the EPT PML4 page. * @param pVCpu The cross context virtual CPU structure. */ DECLINLINE(PEPTPML4) pgmGstGetEptPML4Ptr(PVMCPUCC pVCpu) { PEPTPML4 pEptPml4; int rc = pgmGstGetEptPML4PtrEx(pVCpu, &pEptPml4); AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); return pEptPml4; } # endif #endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ /** * Gets the shadow page directory, 32-bit. * * @returns Pointer to the shadow 32-bit PD. * @param pVCpu The cross context virtual CPU structure. */ DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PVMCPUCC pVCpu) { return (PX86PD)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); } /** * Gets the shadow page directory entry for the specified address, 32-bit. * * @returns Shadow 32-bit PDE. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. */ DECLINLINE(X86PDE) pgmShwGet32BitPDE(PVMCPUCC pVCpu, RTGCPTR GCPtr) { PX86PD pShwPde = pgmShwGet32BitPDPtr(pVCpu); if (!pShwPde) { X86PDE ZeroPde = {0}; return ZeroPde; } return pShwPde->a[(uint32_t)GCPtr >> X86_PD_SHIFT]; } /** * Gets the pointer to the shadow page directory entry for the specified * address, 32-bit. * * @returns Pointer to the shadow 32-bit PDE. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. */ DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr) { PX86PD pPde = pgmShwGet32BitPDPtr(pVCpu); AssertReturn(pPde, NULL); return &pPde->a[(uint32_t)GCPtr >> X86_PD_SHIFT]; } /** * Gets the shadow page pointer table, PAE. * * @returns Pointer to the shadow PAE PDPT. * @param pVCpu The cross context virtual CPU structure. */ DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PVMCPUCC pVCpu) { return (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); } /** * Gets the shadow page directory for the specified address, PAE. * * @returns Pointer to the shadow PD. * @param pVCpu The cross context virtual CPU structure. * @param pPdpt Pointer to the page directory pointer table. * @param GCPtr The address. */ DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPUCC pVCpu, PX86PDPT pPdpt, RTGCPTR GCPtr) { const unsigned iPdpt = (uint32_t)GCPtr >> X86_PDPT_SHIFT; if (pPdpt->a[iPdpt].u & X86_PDPE_P) { /* Fetch the pgm pool shadow descriptor. */ PVMCC pVM = pVCpu->CTX_SUFF(pVM); PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK); AssertReturn(pShwPde, NULL); return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde); } return NULL; } /** * Gets the shadow page directory for the specified address, PAE. * * @returns Pointer to the shadow PD. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. */ DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr) { return pgmShwGetPaePDPtr(pVCpu, pgmShwGetPaePDPTPtr(pVCpu), GCPtr); } /** * Gets the shadow page directory entry, PAE. * * @returns PDE. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. */ DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PVMCPUCC pVCpu, RTGCPTR GCPtr) { const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK; PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pVCpu, GCPtr); if (pShwPde) return pShwPde->a[iPd]; X86PDEPAE ZeroPde = {0}; return ZeroPde; } /** * Gets the pointer to the shadow page directory entry for an address, PAE. * * @returns Pointer to the PDE. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. * @remarks Only used by AssertCR3. */ DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr) { const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK; PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pVCpu, GCPtr); AssertReturn(pShwPde, NULL); return &pShwPde->a[iPd]; } /** * Gets the shadow page map level-4 pointer. * * @returns Pointer to the shadow PML4. * @param pVCpu The cross context virtual CPU structure. */ DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PVMCPUCC pVCpu) { return (PX86PML4)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); } /** * Gets the shadow page map level-4 entry for the specified address. * * @returns The entry. * @param pVCpu The cross context virtual CPU structure. * @param GCPtr The address. */ DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PVMCPUCC pVCpu, RTGCPTR GCPtr) { const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK; PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu); if (pShwPml4) return pShwPml4->a[iPml4]; X86PML4E ZeroPml4e = {0}; return ZeroPml4e; } /** * Gets the pointer to the specified shadow page map level-4 entry. * * @returns The entry. * @param pVCpu The cross context virtual CPU structure. * @param iPml4 The PML4 index. */ DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PVMCPUCC pVCpu, unsigned int iPml4) { PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu); if (pShwPml4) return &pShwPml4->a[iPml4]; return NULL; } #endif /* !VBOX_VMM_TARGET_ARMV8 */ /** * Cached physical handler lookup. * * @returns VBox status code. * @retval VERR_NOT_FOUND if no handler. * @param pVM The cross context VM structure. * @param GCPhys The lookup address. * @param ppHandler Where to return the handler pointer. */ DECLINLINE(int) pgmHandlerPhysicalLookup(PVMCC pVM, RTGCPHYS GCPhys, PPGMPHYSHANDLER *ppHandler) { PPGMPHYSHANDLER pHandler = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.ptrFromInt(pVM->pgm.s.idxLastPhysHandler); if ( pHandler && pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.isPtrRetOkay(pHandler) && GCPhys >= pHandler->Key && GCPhys < pHandler->KeyLast && pHandler->hType != NIL_PGMPHYSHANDLERTYPE && pHandler->hType != 0) { STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysHandlerLookupHits)); *ppHandler = pHandler; return VINF_SUCCESS; } STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysHandlerLookupMisses)); AssertPtrReturn(pVM->VMCC_CTX(pgm).s.pPhysHandlerTree, VERR_PGM_HANDLER_IPE_1); int rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pHandler); if (RT_SUCCESS(rc)) { *ppHandler = pHandler; pVM->pgm.s.idxLastPhysHandler = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.ptrToInt(pHandler); return VINF_SUCCESS; } *ppHandler = NULL; return rc; } /** * Converts a handle to a pointer. * * @returns Pointer on success, NULL on failure (asserted). * @param pVM The cross context VM structure. * @param hType Physical access handler type handle. */ DECLINLINE(PCPGMPHYSHANDLERTYPEINT) pgmHandlerPhysicalTypeHandleToPtr(PVMCC pVM, PGMPHYSHANDLERTYPE hType) { #ifdef IN_RING0 PPGMPHYSHANDLERTYPEINT pType = &pVM->pgmr0.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK]; #elif defined(IN_RING3) PPGMPHYSHANDLERTYPEINT pType = &pVM->pgm.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK]; #else # error "Invalid context" #endif AssertReturn(pType->hType == hType, NULL); return pType; } /** * Converts a handle to a pointer, never returns NULL. * * @returns Pointer on success, dummy on failure (asserted). * @param pVM The cross context VM structure. * @param hType Physical access handler type handle. */ DECLINLINE(PCPGMPHYSHANDLERTYPEINT) pgmHandlerPhysicalTypeHandleToPtr2(PVMCC pVM, PGMPHYSHANDLERTYPE hType) { #ifdef IN_RING0 PPGMPHYSHANDLERTYPEINT pType = &pVM->pgmr0.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK]; #elif defined(IN_RING3) PPGMPHYSHANDLERTYPEINT pType = &pVM->pgm.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK]; #else # error "Invalid context" #endif AssertReturn(pType->hType == hType, &g_pgmHandlerPhysicalDummyType); return pType; } /** * Internal worker for finding a 'in-use' shadow page give by it's physical address. * * @returns Pointer to the shadow page structure. * @param pPool The pool. * @param idx The pool page index. */ DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx) { AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx)); return &pPool->aPages[idx]; } /** * Clear references to guest physical memory. * * @param pPool The pool. * @param pPoolPage The pool page. * @param pPhysPage The physical guest page tracking structure. * @param iPte Shadow PTE index */ DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte) { /* * Just deal with the simple case here. */ #ifdef VBOX_STRICT PVMCC pVM = pPool->CTX_SUFF(pVM); NOREF(pVM); #endif #ifdef LOG_ENABLED const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage); #endif const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage); if (cRefs == 1) { Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage)); Assert(iPte == PGM_PAGE_GET_PTE_INDEX(pPhysPage)); /* Invalidate the tracking data. */ PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0); } else pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage, iPte); Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage )); } /** * Moves the page to the head of the age list. * * This is done when the cached page is used in one way or another. * * @param pPool The pool. * @param pPage The cached page. */ DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage) { PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM)); /* * Move to the head of the age list. */ if (pPage->iAgePrev != NIL_PGMPOOL_IDX) { /* unlink */ pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext; if (pPage->iAgeNext != NIL_PGMPOOL_IDX) pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev; else pPool->iAgeTail = pPage->iAgePrev; /* insert at head */ pPage->iAgePrev = NIL_PGMPOOL_IDX; pPage->iAgeNext = pPool->iAgeHead; Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */ pPool->iAgeHead = pPage->idx; pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx; } } /** * Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages). * * @param pPool The pool. * @param pPage PGM pool page */ DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage) { PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM)); NOREF(pPool); ASMAtomicIncU32(&pPage->cLocked); } /** * Unlocks a page to allow flushing again * * @param pPool The pool. * @param pPage PGM pool page */ DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage) { PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM)); NOREF(pPool); Assert(pPage->cLocked); ASMAtomicDecU32(&pPage->cLocked); } /** * Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT) * * @returns VBox status code. * @param pPage PGM pool page */ DECLINLINE(bool) pgmPoolIsPageLocked(PPGMPOOLPAGE pPage) { if (pPage->cLocked) { LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind)); if (pPage->cModifications) pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */ return true; } return false; } /** * Check if the specified page is dirty (not write monitored) * * @return dirty or not * @param pVM The cross context VM structure. * @param GCPhys Guest physical address */ DECLINLINE(bool) pgmPoolIsDirtyPage(PVMCC pVM, RTGCPHYS GCPhys) { PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); PGM_LOCK_ASSERT_OWNER(pVM); if (!pPool->cDirtyPages) return false; return pgmPoolIsDirtyPageSlow(pVM, GCPhys); } /** @} */ #endif /* !VMM_INCLUDED_SRC_include_PGMInline_h */