Index: trunk/src/VBox/VMM/PGM.cpp
===================================================================
--- trunk/src/VBox/VMM/PGM.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/PGM.cpp	(revision 26150)
@@ -590,4 +590,5 @@
 #include "PGMInternal.h"
 #include <VBox/vm.h>
+#include "PGMInline.h"
 
 #include <VBox/dbg.h>
Index: trunk/src/VBox/VMM/PGMDbg.cpp
===================================================================
--- trunk/src/VBox/VMM/PGMDbg.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/PGMDbg.cpp	(revision 26150)
@@ -28,4 +28,5 @@
 #include "PGMInternal.h"
 #include <VBox/vm.h>
+#include "PGMInline.h"
 #include <iprt/assert.h>
 #include <iprt/asm.h>
@@ -35,4 +36,8 @@
 #include <VBox/err.h>
 
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
 /** The max needle size that we will bother searching for
  * This must not be more than half a page! */
Index: trunk/src/VBox/VMM/PGMHandler.cpp
===================================================================
--- trunk/src/VBox/VMM/PGMHandler.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/PGMHandler.cpp	(revision 26150)
@@ -41,4 +41,5 @@
 #include "PGMInternal.h"
 #include <VBox/vm.h>
+#include "PGMInline.h"
 #include <VBox/dbg.h>
 
Index: trunk/src/VBox/VMM/PGMInline.h
===================================================================
--- trunk/src/VBox/VMM/PGMInline.h	(revision 26150)
+++ trunk/src/VBox/VMM/PGMInline.h	(revision 26150)
@@ -0,0 +1,1641 @@
+/* $Id$ */
+/** @file
+ * PGM - Inlined functions.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Sun Microsystems, Inc.
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
+ * Clara, CA 95054 USA or visit http://www.sun.com if you need
+ * additional information or have any questions.
+ */
+
+#ifndef ___PGMInline_h
+#define ___PGMInline_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/err.h>
+#include <VBox/stam.h>
+#include <VBox/param.h>
+#include <VBox/vmm.h>
+#include <VBox/mm.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmapi.h>
+#include <VBox/dis.h>
+#include <VBox/dbgf.h>
+#include <VBox/log.h>
+#include <VBox/gmm.h>
+#include <VBox/hwaccm.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+#include <iprt/critsect.h>
+#include <iprt/sha.h>
+
+
+
+/** @addtogroup grp_pgm_int   Internals
+ * @internal
+ * @{
+ */
+
+/** @todo Split out all the inline stuff into a separate file.  Then we can
+ *        include it later when VM and VMCPU are defined and so avoid all that
+ *        &pVM->pgm.s and &pVCpu->pgm.s stuff.  It also chops ~1600 lines off
+ *        this file and will make it somewhat easier to navigate... */
+
+/**
+ * Gets the PGMRAMRANGE structure for a guest page.
+ *
+ * @returns Pointer to the RAM range on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+                break;
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    return pRam;
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+                return NULL;
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    return &pRam->aPages[off >> PAGE_SHIFT];
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * Old Phys code: Will make sure the page is present.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS and a valid *ppPage on success.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the page pointer on success.
+ */
+DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+            {
+                *ppPage = NULL; /* avoid incorrect and very annoying GCC warnings */
+                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+            }
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * Old Phys code: Will make sure the page is present.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS and a valid *ppPage on success.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the page pointer on success.
+ * @param   ppRamHint   Where to read and store the ram list hint.
+ *                      The caller initializes this to NULL before the call.
+ */
+DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
+{
+    RTGCPHYS off;
+    PPGMRAMRANGE pRam = *ppRamHint;
+    if (    !pRam
+        ||  RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
+    {
+        pRam = pPGM->CTX_SUFF(pRamRanges);
+        off = GCPhys - pRam->GCPhys;
+        if (RT_UNLIKELY(off >= pRam->cb))
+        {
+            do
+            {
+                pRam = pRam->CTX_SUFF(pNext);
+                if (RT_UNLIKELY(!pRam))
+                {
+                    *ppPage = NULL; /* Kill the incorrect and extremely annoying GCC warnings. */
+                    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+                }
+                off = GCPhys - pRam->GCPhys;
+            } while (off >= pRam->cb);
+        }
+        *ppRamHint = pRam;
+    }
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppRam       Where to store the pointer to the PGMRAMRANGE.
+ */
+DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+                return NULL;
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    *ppRam = pRam;
+    return &pRam->aPages[off >> PAGE_SHIFT];
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the pointer to the PGMPAGE structure.
+ * @param   ppRam       Where to store the pointer to the PGMRAMRANGE structure.
+ */
+DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+            {
+                *ppRam = NULL;  /* Shut up silly GCC warnings. */
+                *ppPage = NULL; /* ditto */
+                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+            }
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    *ppRam = pRam;
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Convert GC Phys to HC Phys.
+ *
+ * @returns VBox status.
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   pHCPhys     Where to store the corresponding HC physical address.
+ *
+ * @deprecated  Doesn't deal with zero, shared or write monitored pages.
+ *              Avoid when writing new code!
+ */
+DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
+{
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
+    if (RT_FAILURE(rc))
+        return rc;
+    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
+    return VINF_SUCCESS;
+}
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+
+/**
+ * Inlined version of the ring-0 version of PGMDynMapHCPage that
+ * optimizes access to pages already in the set.
+ *
+ * @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
+ * @param   pPGM        Pointer to the PVM instance data.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
+{
+    PVM         pVM     = PGM2VM(pPGM);
+    PPGMCPU     pPGMCPU = (PPGMCPU)((uint8_t *)VMMGetCpu(pVM) + pPGM->offVCpuPGM); /* very pretty ;-) */
+    PPGMMAPSET  pSet    = &pPGMCPU->AutoSet;
+
+    STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
+    {
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
+        pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
+    }
+
+    STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
+ * access to pages already in the set.
+ *
+ * @returns See PGMDynMapGCPage.
+ * @param   pPGM        Pointer to the PVM instance data.
+ * @param   GCPhys      The guest physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
+{
+    PVM     pVM     = PGM2VM(pPGM);
+    PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t *)VMMGetCpu(pVM) + pPGM->offVCpuPGM); /* very pretty ;-) */
+
+    STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
+    AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys));
+
+    /*
+     * Get the ram range.
+     */
+    PPGMRAMRANGE    pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS        off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb
+        /** @todo   || page state stuff */))
+    {
+        /* This case is not counted into StatR0DynMapGCPageInl. */
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
+        return PGMDynMapGCPage(pVM, GCPhys, ppv);
+    }
+
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
+    STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
+
+    /*
+     * pgmR0DynMapHCPageInlined with out stats.
+     */
+    PPGMMAPSET pSet = &pPGMCPU->AutoSet;
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
+    {
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
+        pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
+    }
+
+    STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined version of the ring-0 version of PGMDynMapGCPageOff that optimizes
+ * access to pages already in the set.
+ *
+ * @returns See PGMDynMapGCPage.
+ * @param   pPGM        Pointer to the PVM instance data.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmR0DynMapGCPageOffInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
+{
+    PVM     pVM     = PGM2VM(pPGM);
+    PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t *)VMMGetCpu(pVM) + pPGM->offVCpuPGM); /* very pretty ;-) */
+
+    STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
+
+    /*
+     * Get the ram range.
+     */
+    PPGMRAMRANGE    pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS        off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb
+        /** @todo   || page state stuff */))
+    {
+        /* This case is not counted into StatR0DynMapGCPageInl. */
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
+        return PGMDynMapGCPageOff(pVM, GCPhys, ppv);
+    }
+
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
+    STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
+
+    /*
+     * pgmR0DynMapHCPageInlined with out stats.
+     */
+    PPGMMAPSET pSet = &pPGMCPU->AutoSet;
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
+    {
+        *ppv = (void *)((uintptr_t)pSet->aEntries[iEntry].pvPage | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
+        pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
+        *ppv = (void *)((uintptr_t)*ppv | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
+    }
+
+    STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+
+/**
+ * Maps the page into current context (RC and maybe R0).
+ *
+ * @returns pointer to the mapping.
+ * @param   pVM         Pointer to the PGM instance data.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
+{
+    if (pPage->idx >= PGMPOOL_IDX_FIRST)
+    {
+        Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
+        void *pv;
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+        pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
+# else
+        PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
+# endif
+        return pv;
+    }
+    AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
+}
+
+/**
+ * Temporarily maps one host page specified by HC physical address, returning
+ * pointer within the page.
+ *
+ * Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
+ * reused after 8 mappings (or perhaps a few more if you score with the cache).
+ *
+ * @returns The address corresponding to HCPhys.
+ * @param   pPGM        Pointer to the PVM instance data.
+ * @param   HCPhys      HC Physical address of the page.
+ */
+DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
+{
+    void *pv;
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
+# else
+    PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
+# endif
+    pv = (void *)((uintptr_t)pv | ((uintptr_t)HCPhys & PAGE_OFFSET_MASK));
+    return pv;
+}
+
+#endif /*  VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 || IN_RC */
+#ifndef IN_RC
+
+/**
+ * Queries the Physical TLB entry for a physical guest page,
+ * attempting to load the TLB entry if necessary.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pPGM        The PGM instance handle.
+ * @param   GCPhys      The address of the guest page.
+ * @param   ppTlbe      Where to store the pointer to the TLB entry.
+ */
+DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+{
+    int rc;
+    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
+    {
+        STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
+    *ppTlbe = pTlbe;
+    return rc;
+}
+
+
+/**
+ * Queries the Physical TLB entry for a physical guest page,
+ * attempting to load the TLB entry if necessary.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pPGM        The PGM instance handle.
+ * @param   pPage       Pointer to the PGMPAGE structure corresponding to
+ *                      GCPhys.
+ * @param   GCPhys      The address of the guest page.
+ * @param   ppTlbe      Where to store the pointer to the TLB entry.
+ */
+DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+{
+    int rc;
+    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
+    {
+        STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
+    *ppTlbe = pTlbe;
+    return rc;
+}
+
+#endif /* !IN_RC */
+
+/**
+ * Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
+ * Takes PSE-36 into account.
+ *
+ * @returns guest physical address
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   Pde         Guest Pde
+ */
+DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
+{
+    RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
+    GCPhys |= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
+
+    return GCPhys & pPGM->GCPhys4MBPSEMask;
+}
+
+
+/**
+ * Gets the page directory entry for the specified address (32-bit paging).
+ *
+ * @returns The page directory entry in question.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PCX86PD pGuestPD = NULL;
+    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
+    if (RT_FAILURE(rc))
+    {
+        X86PDE ZeroPde = {0};
+        AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
+    }
+#else
+    PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
+# ifdef IN_RING3
+    if (!pGuestPD)
+        pGuestPD = pgmGstLazyMap32BitPD(pPGM);
+# endif
+#endif
+    return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
+}
+
+
+/**
+ * Gets the address of a specific page directory entry (32-bit paging).
+ *
+ * @returns Pointer the page directory entry in question.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PD  pGuestPD = NULL;
+    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
+# ifdef IN_RING3
+    if (!pGuestPD)
+        pGuestPD = pgmGstLazyMap32BitPD(pPGM);
+# endif
+#endif
+    return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
+}
+
+
+/**
+ * Gets the address the guest page directory (32-bit paging).
+ *
+ * @returns Pointer the page directory entry in question.
+ * @param   pPGM        Pointer to the PGM instance data.
+ */
+DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PD  pGuestPD = NULL;
+    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
+# ifdef IN_RING3
+    if (!pGuestPD)
+        pGuestPD = pgmGstLazyMap32BitPD(pPGM);
+# endif
+#endif
+    return pGuestPD;
+}
+
+
+/**
+ * Gets the guest page directory pointer table.
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ */
+DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PDPT pGuestPDPT = NULL;
+    int rc = pgmR0DynMapGCPageOffInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
+# ifdef IN_RING3
+    if (!pGuestPDPT)
+        pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
+# endif
+#endif
+    return pGuestPDPT;
+}
+
+
+/**
+ * Gets the guest page directory pointer table entry for the specified address.
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PDPT pGuestPDPT = 0;
+    int rc = pgmR0DynMapGCPageOffInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
+    AssertRCReturn(rc, 0);
+#else
+    PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
+# ifdef IN_RING3
+    if (!pGuestPDPT)
+        pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
+# endif
+#endif
+    return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
+}
+
+
+/**
+ * Gets the page directory for the specified address.
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+
+    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
+    AssertReturn(pGuestPDPT, NULL);
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    if (pGuestPDPT->a[iPdpt].n.u1Present)
+    {
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+        PX86PDPAE   pGuestPD = NULL;
+        int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
+        AssertRCReturn(rc, NULL);
+#else
+        PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
+        if (    !pGuestPD
+            ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
+            pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
+#endif
+        return pGuestPD;
+        /* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns Pointer to the page directory entry in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+
+    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
+    AssertReturn(pGuestPDPT, NULL);
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    if (pGuestPDPT->a[iPdpt].n.u1Present)
+    {
+        const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+        PX86PDPAE   pGuestPD = NULL;
+        int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
+        AssertRCReturn(rc, NULL);
+#else
+        PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
+        if (    !pGuestPD
+            ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
+            pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
+#endif
+        return &pGuestPD->a[iPD];
+        /* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns The page directory entry in question.
+ * @returns A non-present entry if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+    X86PDEPAE   ZeroPde = {0};
+    PX86PDPT    pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
+    if (RT_LIKELY(pGuestPDPT))
+    {
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+        if (pGuestPDPT->a[iPdpt].n.u1Present)
+        {
+            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+            PX86PDPAE   pGuestPD = NULL;
+            int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
+            AssertRCReturn(rc, ZeroPde);
+#else
+            PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
+            if (    !pGuestPD
+                ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
+                pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
+#endif
+            return pGuestPD->a[iPD];
+        }
+    }
+    return ZeroPde;
+}
+
+
+/**
+ * Gets the page directory pointer table entry for the specified address
+ * and returns the index into the page directory
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ * @param   piPD        Receives the index into the returned page directory
+ * @param   pPdpe       Receives the page directory pointer entry. Optional.
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
+{
+    AssertGCPtr32(GCPtr);
+
+    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
+    AssertReturn(pGuestPDPT, NULL);
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    if (pPdpe)
+        *pPdpe = pGuestPDPT->a[iPdpt];
+    if (pGuestPDPT->a[iPdpt].n.u1Present)
+    {
+        const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+        PX86PDPAE   pGuestPD = NULL;
+        int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
+        AssertRCReturn(rc, NULL);
+#else
+        PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
+        if (    !pGuestPD
+            ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
+            pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
+#endif
+        *piPD = iPD;
+        return pGuestPD;
+        /* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
+    }
+    return NULL;
+}
+
+#ifndef IN_RC
+
+/**
+ * Gets the page map level-4 pointer for the guest.
+ *
+ * @returns Pointer to the PML4 page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ */
+DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PML4 pGuestPml4;
+    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
+    if (!pGuestPml4)
+        pGuestPml4 = pgmGstLazyMapPml4(pPGM);
+# endif
+    Assert(pGuestPml4);
+#endif
+    return pGuestPml4;
+}
+
+
+/**
+ * Gets the pointer to a page map level-4 entry.
+ *
+ * @returns Pointer to the PML4 entry.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   iPml4       The index.
+ */
+DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PML4 pGuestPml4;
+    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
+    if (!pGuestPml4)
+        pGuestPml4 = pgmGstLazyMapPml4(pPGM);
+# endif
+    Assert(pGuestPml4);
+#endif
+    return &pGuestPml4->a[iPml4];
+}
+
+
+/**
+ * Gets a page map level-4 entry.
+ *
+ * @returns The PML4 entry.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   iPml4       The index.
+ */
+DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PML4 pGuestPml4;
+    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
+    if (RT_FAILURE(rc))
+    {
+        X86PML4E ZeroPml4e = {0};
+        AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
+    }
+#else
+    PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
+    if (!pGuestPml4)
+        pGuestPml4 = pgmGstLazyMapPml4(pPGM);
+# endif
+    Assert(pGuestPml4);
+#endif
+    return pGuestPml4->a[iPml4];
+}
+
+
+/**
+ * Gets the page directory pointer entry for the specified address.
+ *
+ * @returns Pointer to the page directory pointer entry in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ * @param   ppPml4e     Page Map Level-4 Entry (out)
+ */
+DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
+{
+    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
+    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PCX86PML4E      pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
+    if (pPml4e->n.u1Present)
+    {
+        PX86PDPT pPdpt;
+        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
+        AssertRCReturn(rc, NULL);
+
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+        return &pPdpt->a[iPdpt];
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns The page directory entry in question.
+ * @returns A non-present entry if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ * @param   ppPml4e     Page Map Level-4 Entry (out)
+ * @param   pPdpe       Page directory pointer table entry (out)
+ */
+DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
+{
+    X86PDEPAE       ZeroPde = {0};
+    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
+    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PCX86PML4E      pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
+    if (pPml4e->n.u1Present)
+    {
+        PCX86PDPT   pPdptTemp;
+        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
+        AssertRCReturn(rc, ZeroPde);
+
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+        *pPdpe = pPdptTemp->a[iPdpt];
+        if (pPdptTemp->a[iPdpt].n.u1Present)
+        {
+            PCX86PDPAE pPD;
+            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+            AssertRCReturn(rc, ZeroPde);
+
+            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+            return pPD->a[iPD];
+        }
+    }
+
+    return ZeroPde;
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns The page directory entry in question.
+ * @returns A non-present entry if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
+{
+    X86PDEPAE       ZeroPde = {0};
+    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
+    const unsigned  iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (pGuestPml4->a[iPml4].n.u1Present)
+    {
+        PCX86PDPT   pPdptTemp;
+        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        AssertRCReturn(rc, ZeroPde);
+
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+        if (pPdptTemp->a[iPdpt].n.u1Present)
+        {
+            PCX86PDPAE pPD;
+            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+            AssertRCReturn(rc, ZeroPde);
+
+            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+            return pPD->a[iPD];
+        }
+    }
+    return ZeroPde;
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns Pointer to the page directory entry in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
+{
+    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
+    const unsigned  iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (pGuestPml4->a[iPml4].n.u1Present)
+    {
+        PCX86PDPT   pPdptTemp;
+        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        AssertRCReturn(rc, NULL);
+
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+        if (pPdptTemp->a[iPdpt].n.u1Present)
+        {
+            PX86PDPAE pPD;
+            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+            AssertRCReturn(rc, NULL);
+
+            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+            return &pPD->a[iPD];
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the GUEST page directory pointer for the specified address.
+ *
+ * @returns The page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ * @param   ppPml4e     Page Map Level-4 Entry (out)
+ * @param   pPdpe       Page directory pointer table entry (out)
+ * @param   piPD        Receives the index into the returned page directory
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe, unsigned *piPD)
+{
+    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
+    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PCX86PML4E      pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
+    if (pPml4e->n.u1Present)
+    {
+        PCX86PDPT   pPdptTemp;
+        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
+        AssertRCReturn(rc, NULL);
+
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+        *pPdpe = pPdptTemp->a[iPdpt];
+        if (pPdptTemp->a[iPdpt].n.u1Present)
+        {
+            PX86PDPAE pPD;
+            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+            AssertRCReturn(rc, NULL);
+
+            *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+            return pPD;
+        }
+    }
+    return 0;
+}
+
+#endif /* !IN_RC */
+
+/**
+ * Gets the shadow page directory, 32-bit.
+ *
+ * @returns Pointer to the shadow 32-bit PD.
+ * @param   pPGM        Pointer to the PGM instance data.
+ */
+DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
+{
+    return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page directory entry for the specified address, 32-bit.
+ *
+ * @returns Shadow 32-bit PDE.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
+
+    PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
+    if (!pShwPde)
+    {
+        X86PDE ZeroPde = {0};
+        return ZeroPde;
+    }
+    return pShwPde->a[iPd];
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for the specified
+ * address, 32-bit.
+ *
+ * @returns Pointer to the shadow 32-bit PDE.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
+
+    PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
+    AssertReturn(pPde, NULL);
+    return &pPde->a[iPd];
+}
+
+
+/**
+ * Gets the shadow page pointer table, PAE.
+ *
+ * @returns Pointer to the shadow PAE PDPT.
+ * @param   pPGM        Pointer to the PGM instance data.
+ */
+DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
+{
+    return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    PX86PDPT        pPdpt = pgmShwGetPaePDPTPtr(pPGM);
+
+    if (!pPdpt->a[iPdpt].n.u1Present)
+        return NULL;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, NULL);
+
+    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
+{
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+
+    if (!pPdpt->a[iPdpt].n.u1Present)
+        return NULL;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, NULL);
+
+    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
+}
+
+
+/**
+ * Gets the shadow page directory entry, PAE.
+ *
+ * @returns PDE.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    const unsigned  iPd   = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+
+    PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
+    if (!pShwPde)
+    {
+        X86PDEPAE ZeroPde = {0};
+        return ZeroPde;
+    }
+    return pShwPde->a[iPd];
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for an address, PAE.
+ *
+ * @returns Pointer to the PDE.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    const unsigned  iPd   = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+
+    PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
+    AssertReturn(pPde, NULL);
+    return &pPde->a[iPd];
+}
+
+#ifndef IN_RC
+
+/**
+ * Gets the shadow page map level-4 pointer.
+ *
+ * @returns Pointer to the shadow PML4.
+ * @param   pPGM        Pointer to the PGM instance data.
+ */
+DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
+{
+    return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page map level-4 entry for the specified address.
+ *
+ * @returns The entry.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
+{
+    const unsigned  iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PX86PML4        pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
+
+    if (!pShwPml4)
+    {
+        X86PML4E ZeroPml4e = {0};
+        return ZeroPml4e;
+    }
+    return pShwPml4->a[iPml4];
+}
+
+
+/**
+ * Gets the pointer to the specified shadow page map level-4 entry.
+ *
+ * @returns The entry.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   iPml4       The PML4 index.
+ */
+DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
+{
+    PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
+    if (!pShwPml4)
+        return NULL;
+    return &pShwPml4->a[iPml4];
+}
+
+
+/**
+ * Gets the GUEST page directory pointer for the specified address.
+ *
+ * @returns The page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   GCPtr       The address.
+ * @param   piPD        Receives the index into the returned page directory
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
+{
+    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
+    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (pGuestPml4->a[iPml4].n.u1Present)
+    {
+        PCX86PDPT   pPdptTemp;
+        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        AssertRCReturn(rc, NULL);
+
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+        if (pPdptTemp->a[iPdpt].n.u1Present)
+        {
+            PX86PDPAE pPD;
+            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+            AssertRCReturn(rc, NULL);
+
+            *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+            return pPD;
+        }
+    }
+    return NULL;
+}
+
+#endif /* !IN_RC */
+
+/**
+ * Gets the page state for a physical handler.
+ *
+ * @returns The physical handler page state.
+ * @param   pCur    The physical handler in question.
+ */
+DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
+{
+    switch (pCur->enmType)
+    {
+        case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
+            return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
+
+        case PGMPHYSHANDLERTYPE_MMIO:
+        case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
+            return PGM_PAGE_HNDL_PHYS_STATE_ALL;
+
+        default:
+            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
+    }
+}
+
+
+/**
+ * Gets the page state for a virtual handler.
+ *
+ * @returns The virtual handler page state.
+ * @param   pCur    The virtual handler in question.
+ * @remarks This should never be used on a hypervisor access handler.
+ */
+DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
+{
+    switch (pCur->enmType)
+    {
+        case PGMVIRTHANDLERTYPE_WRITE:
+            return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
+        case PGMVIRTHANDLERTYPE_ALL:
+            return PGM_PAGE_HNDL_VIRT_STATE_ALL;
+        default:
+            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
+    }
+}
+
+
+/**
+ * Clears one physical page of a virtual handler
+ *
+ * @param   pPGM    Pointer to the PGM instance.
+ * @param   pCur    Virtual handler structure
+ * @param   iPage   Physical page index
+ *
+ * @remark  Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
+ *          need to care about other handlers in the same page.
+ */
+DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
+{
+    const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
+
+    /*
+     * Remove the node from the tree (it's supposed to be in the tree if we get here!).
+     */
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+    AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
+                     ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                      pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
+#endif
+    if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
+    {
+        /* We're the head of the alias chain. */
+        PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+        AssertReleaseMsg(pRemove != NULL,
+                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
+        AssertReleaseMsg(pRemove == pPhys2Virt,
+                         ("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
+                          "   got:    pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
+                          pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
+#endif
+        if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
+        {
+            /* Insert the next list in the alias chain into the tree. */
+            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+            AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
+                             ("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                             pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
+#endif
+            pNext->offNextAlias |= PGMPHYS2VIRTHANDLER_IS_HEAD;
+            bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
+            AssertRelease(fRc);
+        }
+    }
+    else
+    {
+        /* Locate the previous node in the alias chain. */
+        PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+        AssertReleaseMsg(pPrev != pPhys2Virt,
+                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
+                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
+#endif
+        for (;;)
+        {
+            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
+            if (pNext == pPhys2Virt)
+            {
+                /* unlink. */
+                LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
+                         pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
+                if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
+                    pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
+                else
+                {
+                    PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
+                    pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
+                                        | (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
+                }
+                break;
+            }
+
+            /* next */
+            if (pNext == pPrev)
+            {
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+                AssertReleaseMsg(pNext != pPrev,
+                                 ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
+                                  pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
+#endif
+                break;
+            }
+            pPrev = pNext;
+        }
+    }
+    Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
+          pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
+    pPhys2Virt->offNextAlias = 0;
+    pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
+
+    /*
+     * Clear the ram flags for this page.
+     */
+    PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
+    AssertReturnVoid(pPage);
+    PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
+}
+
+
+/**
+ * Internal worker for finding a 'in-use' shadow page give by it's physical address.
+ *
+ * @returns Pointer to the shadow page structure.
+ * @param   pPool       The pool.
+ * @param   idx         The pool page index.
+ */
+DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
+{
+    AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
+    return &pPool->aPages[idx];
+}
+
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPoolPage   The pool page.
+ * @param   pPhysPage   The physical guest page tracking structure.
+ */
+DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
+{
+    /*
+     * Just deal with the simple case here.
+     */
+# ifdef LOG_ENABLED
+    const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
+# endif
+    const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
+    if (cRefs == 1)
+    {
+        Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
+        PGM_PAGE_SET_TRACKING(pPhysPage, 0);
+    }
+    else
+        pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
+    Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
+}
+
+
+/**
+ * Moves the page to the head of the age list.
+ *
+ * This is done when the cached page is used in one way or another.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    PVM pVM = pPool->CTX_SUFF(pVM);
+    pgmLock(pVM);
+
+    /*
+     * Move to the head of the age list.
+     */
+    if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
+    {
+        /* unlink */
+        pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
+        if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
+            pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
+        else
+            pPool->iAgeTail = pPage->iAgePrev;
+
+        /* insert at head */
+        pPage->iAgePrev = NIL_PGMPOOL_IDX;
+        pPage->iAgeNext = pPool->iAgeHead;
+        Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
+        pPool->iAgeHead = pPage->idx;
+        pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
+    }
+    pgmUnlock(pVM);
+}
+
+/**
+ * Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
+ *
+ * @param   pVM         VM Handle.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
+    ASMAtomicIncU32(&pPage->cLocked);
+}
+
+
+/**
+ * Unlocks a page to allow flushing again
+ *
+ * @param   pVM         VM Handle.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
+    Assert(pPage->cLocked);
+    ASMAtomicDecU32(&pPage->cLocked);
+}
+
+
+/**
+ * Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
+ *
+ * @returns VBox status code.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
+{
+    if (pPage->cLocked)
+    {
+        LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
+        if (pPage->cModifications)
+            pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Tells if mappings are to be put into the shadow page table or not.
+ *
+ * @returns boolean result
+ * @param   pVM         VM handle.
+ */
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
+{
+#ifdef PGM_WITHOUT_MAPPINGS
+    /* There are no mappings in VT-x and AMD-V mode. */
+    Assert(pPGM->fMappingsDisabled);
+    return false;
+#else
+    return !pPGM->fMappingsDisabled;
+#endif
+}
+
+
+/**
+ * Checks if the mappings are floating and enabled.
+ *
+ * @returns true / false.
+ * @param   pVM         The VM handle.
+ */
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PPGM pPGM)
+{
+#ifdef PGM_WITHOUT_MAPPINGS
+    /* There are no mappings in VT-x and AMD-V mode. */
+    Assert(pPGM->fMappingsDisabled);
+    return false;
+#else
+    return !pPGM->fMappingsDisabled
+        && !pPGM->fMappingsFixed;
+#endif
+}
+
+/** @} */
+
+#endif
+
Index: trunk/src/VBox/VMM/PGMInternal.h
===================================================================
--- trunk/src/VBox/VMM/PGMInternal.h	(revision 26149)
+++ trunk/src/VBox/VMM/PGMInternal.h	(revision 26150)
@@ -3360,1590 +3360,4 @@
 RT_C_DECLS_END
 
-/** @todo Split out all the inline stuff into a separate file.  Then we can
- *        include it later when VM and VMCPU are defined and so avoid all that
- *        &pVM->pgm.s and &pVCpu->pgm.s stuff.  It also chops ~1600 lines off
- *        this file and will make it somewhat easier to navigate... */
-
-/**
- * Gets the PGMRAMRANGE structure for a guest page.
- *
- * @returns Pointer to the RAM range on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- */
-DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-                break;
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    return pRam;
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page.
- *
- * @returns Pointer to the page on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- */
-DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-                return NULL;
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    return &pRam->aPages[off >> PAGE_SHIFT];
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page.
- *
- * Old Phys code: Will make sure the page is present.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS and a valid *ppPage on success.
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppPage      Where to store the page pointer on success.
- */
-DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-            {
-                *ppPage = NULL; /* avoid incorrect and very annoying GCC warnings */
-                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-            }
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
-    return VINF_SUCCESS;
-}
-
-
-
-
-/**
- * Gets the PGMPAGE structure for a guest page.
- *
- * Old Phys code: Will make sure the page is present.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS and a valid *ppPage on success.
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppPage      Where to store the page pointer on success.
- * @param   ppRamHint   Where to read and store the ram list hint.
- *                      The caller initializes this to NULL before the call.
- */
-DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
-{
-    RTGCPHYS off;
-    PPGMRAMRANGE pRam = *ppRamHint;
-    if (    !pRam
-        ||  RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
-    {
-        pRam = pPGM->CTX_SUFF(pRamRanges);
-        off = GCPhys - pRam->GCPhys;
-        if (RT_UNLIKELY(off >= pRam->cb))
-        {
-            do
-            {
-                pRam = pRam->CTX_SUFF(pNext);
-                if (RT_UNLIKELY(!pRam))
-                {
-                    *ppPage = NULL; /* Kill the incorrect and extremely annoying GCC warnings. */
-                    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-                }
-                off = GCPhys - pRam->GCPhys;
-            } while (off >= pRam->cb);
-        }
-        *ppRamHint = pRam;
-    }
-    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
- *
- * @returns Pointer to the page on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppRam       Where to store the pointer to the PGMRAMRANGE.
- */
-DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-                return NULL;
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    *ppRam = pRam;
-    return &pRam->aPages[off >> PAGE_SHIFT];
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
- *
- * @returns Pointer to the page on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppPage      Where to store the pointer to the PGMPAGE structure.
- * @param   ppRam       Where to store the pointer to the PGMRAMRANGE structure.
- */
-DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-            {
-                *ppRam = NULL;  /* Shut up silly GCC warnings. */
-                *ppPage = NULL; /* ditto */
-                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-            }
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    *ppRam = pRam;
-    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Convert GC Phys to HC Phys.
- *
- * @returns VBox status.
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   pHCPhys     Where to store the corresponding HC physical address.
- *
- * @deprecated  Doesn't deal with zero, shared or write monitored pages.
- *              Avoid when writing new code!
- */
-DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
-{
-    PPGMPAGE pPage;
-    int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
-    if (RT_FAILURE(rc))
-        return rc;
-    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
-    return VINF_SUCCESS;
-}
-
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-
-/**
- * Inlined version of the ring-0 version of PGMDynMapHCPage that
- * optimizes access to pages already in the set.
- *
- * @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
- * @param   pPGM        Pointer to the PVM instance data.
- * @param   HCPhys      The physical address of the page.
- * @param   ppv         Where to store the mapping address.
- */
-DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
-{
-    PVM         pVM     = PGM2VM(pPGM);
-    PPGMCPU     pPGMCPU = (PPGMCPU)((uint8_t *)VMMGetCpu(pVM) + pPGM->offVCpuPGM); /* very pretty ;-) */
-    PPGMMAPSET  pSet    = &pPGMCPU->AutoSet;
-
-    STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
-    Assert(!(HCPhys & PAGE_OFFSET_MASK));
-    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
-
-    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
-    unsigned    iEntry  = pSet->aiHashTable[iHash];
-    if (    iEntry < pSet->cEntries
-        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
-    {
-        *ppv = pSet->aEntries[iEntry].pvPage;
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
-    }
-    else
-    {
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
-        pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
-    }
-
-    STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
- * access to pages already in the set.
- *
- * @returns See PGMDynMapGCPage.
- * @param   pPGM        Pointer to the PVM instance data.
- * @param   GCPhys      The guest physical address of the page.
- * @param   ppv         Where to store the mapping address.
- */
-DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
-{
-    PVM     pVM     = PGM2VM(pPGM);
-    PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t *)VMMGetCpu(pVM) + pPGM->offVCpuPGM); /* very pretty ;-) */
-
-    STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
-    AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys));
-
-    /*
-     * Get the ram range.
-     */
-    PPGMRAMRANGE    pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS        off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb
-        /** @todo   || page state stuff */))
-    {
-        /* This case is not counted into StatR0DynMapGCPageInl. */
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
-        return PGMDynMapGCPage(pVM, GCPhys, ppv);
-    }
-
-    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
-    STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
-
-    /*
-     * pgmR0DynMapHCPageInlined with out stats.
-     */
-    PPGMMAPSET pSet = &pPGMCPU->AutoSet;
-    Assert(!(HCPhys & PAGE_OFFSET_MASK));
-    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
-
-    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
-    unsigned    iEntry  = pSet->aiHashTable[iHash];
-    if (    iEntry < pSet->cEntries
-        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
-    {
-        *ppv = pSet->aEntries[iEntry].pvPage;
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
-    }
-    else
-    {
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
-        pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
-    }
-
-    STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Inlined version of the ring-0 version of PGMDynMapGCPageOff that optimizes
- * access to pages already in the set.
- *
- * @returns See PGMDynMapGCPage.
- * @param   pPGM        Pointer to the PVM instance data.
- * @param   HCPhys      The physical address of the page.
- * @param   ppv         Where to store the mapping address.
- */
-DECLINLINE(int) pgmR0DynMapGCPageOffInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
-{
-    PVM     pVM     = PGM2VM(pPGM);
-    PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t *)VMMGetCpu(pVM) + pPGM->offVCpuPGM); /* very pretty ;-) */
-
-    STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
-
-    /*
-     * Get the ram range.
-     */
-    PPGMRAMRANGE    pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS        off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb
-        /** @todo   || page state stuff */))
-    {
-        /* This case is not counted into StatR0DynMapGCPageInl. */
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
-        return PGMDynMapGCPageOff(pVM, GCPhys, ppv);
-    }
-
-    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
-    STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
-
-    /*
-     * pgmR0DynMapHCPageInlined with out stats.
-     */
-    PPGMMAPSET pSet = &pPGMCPU->AutoSet;
-    Assert(!(HCPhys & PAGE_OFFSET_MASK));
-    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
-
-    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
-    unsigned    iEntry  = pSet->aiHashTable[iHash];
-    if (    iEntry < pSet->cEntries
-        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
-    {
-        *ppv = (void *)((uintptr_t)pSet->aEntries[iEntry].pvPage | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
-    }
-    else
-    {
-        STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
-        pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
-        *ppv = (void *)((uintptr_t)*ppv | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
-    }
-
-    STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
-    return VINF_SUCCESS;
-}
-
-#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
-
-/**
- * Maps the page into current context (RC and maybe R0).
- *
- * @returns pointer to the mapping.
- * @param   pVM         Pointer to the PGM instance data.
- * @param   pPage       The page.
- */
-DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
-{
-    if (pPage->idx >= PGMPOOL_IDX_FIRST)
-    {
-        Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
-        void *pv;
-# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-        pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
-# else
-        PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
-# endif
-        return pv;
-    }
-    AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
-}
-
-/**
- * Temporarily maps one host page specified by HC physical address, returning
- * pointer within the page.
- *
- * Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
- * reused after 8 mappings (or perhaps a few more if you score with the cache).
- *
- * @returns The address corresponding to HCPhys.
- * @param   pPGM        Pointer to the PVM instance data.
- * @param   HCPhys      HC Physical address of the page.
- */
-DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
-{
-    void *pv;
-# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
-# else
-    PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
-# endif
-    pv = (void *)((uintptr_t)pv | ((uintptr_t)HCPhys & PAGE_OFFSET_MASK));
-    return pv;
-}
-
-#endif /*  VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 || IN_RC */
-#ifndef IN_RC
-
-/**
- * Queries the Physical TLB entry for a physical guest page,
- * attempting to load the TLB entry if necessary.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
- *
- * @param   pPGM        The PGM instance handle.
- * @param   GCPhys      The address of the guest page.
- * @param   ppTlbe      Where to store the pointer to the TLB entry.
- */
-DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
-{
-    int rc;
-    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
-    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
-    {
-        STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
-        rc = VINF_SUCCESS;
-    }
-    else
-        rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
-    *ppTlbe = pTlbe;
-    return rc;
-}
-
-
-/**
- * Queries the Physical TLB entry for a physical guest page,
- * attempting to load the TLB entry if necessary.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
- *
- * @param   pPGM        The PGM instance handle.
- * @param   pPage       Pointer to the PGMPAGE structure corresponding to
- *                      GCPhys.
- * @param   GCPhys      The address of the guest page.
- * @param   ppTlbe      Where to store the pointer to the TLB entry.
- */
-DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
-{
-    int rc;
-    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
-    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
-    {
-        STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
-        rc = VINF_SUCCESS;
-    }
-    else
-        rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
-    *ppTlbe = pTlbe;
-    return rc;
-}
-
-#endif /* !IN_RC */
-
-/**
- * Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
- * Takes PSE-36 into account.
- *
- * @returns guest physical address
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   Pde         Guest Pde
- */
-DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
-{
-    RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
-    GCPhys |= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
-
-    return GCPhys & pPGM->GCPhys4MBPSEMask;
-}
-
-
-/**
- * Gets the page directory entry for the specified address (32-bit paging).
- *
- * @returns The page directory entry in question.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PCX86PD pGuestPD = NULL;
-    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
-    if (RT_FAILURE(rc))
-    {
-        X86PDE ZeroPde = {0};
-        AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
-    }
-#else
-    PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
-# ifdef IN_RING3
-    if (!pGuestPD)
-        pGuestPD = pgmGstLazyMap32BitPD(pPGM);
-# endif
-#endif
-    return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
-}
-
-
-/**
- * Gets the address of a specific page directory entry (32-bit paging).
- *
- * @returns Pointer the page directory entry in question.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PD  pGuestPD = NULL;
-    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
-    AssertRCReturn(rc, NULL);
-#else
-    PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
-# ifdef IN_RING3
-    if (!pGuestPD)
-        pGuestPD = pgmGstLazyMap32BitPD(pPGM);
-# endif
-#endif
-    return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
-}
-
-
-/**
- * Gets the address the guest page directory (32-bit paging).
- *
- * @returns Pointer the page directory entry in question.
- * @param   pPGM        Pointer to the PGM instance data.
- */
-DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PD  pGuestPD = NULL;
-    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
-    AssertRCReturn(rc, NULL);
-#else
-    PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
-# ifdef IN_RING3
-    if (!pGuestPD)
-        pGuestPD = pgmGstLazyMap32BitPD(pPGM);
-# endif
-#endif
-    return pGuestPD;
-}
-
-
-/**
- * Gets the guest page directory pointer table.
- *
- * @returns Pointer to the page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- */
-DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PDPT pGuestPDPT = NULL;
-    int rc = pgmR0DynMapGCPageOffInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
-    AssertRCReturn(rc, NULL);
-#else
-    PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
-# ifdef IN_RING3
-    if (!pGuestPDPT)
-        pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
-# endif
-#endif
-    return pGuestPDPT;
-}
-
-
-/**
- * Gets the guest page directory pointer table entry for the specified address.
- *
- * @returns Pointer to the page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    AssertGCPtr32(GCPtr);
-
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PDPT pGuestPDPT = 0;
-    int rc = pgmR0DynMapGCPageOffInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
-    AssertRCReturn(rc, 0);
-#else
-    PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
-# ifdef IN_RING3
-    if (!pGuestPDPT)
-        pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
-# endif
-#endif
-    return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
-}
-
-
-/**
- * Gets the page directory for the specified address.
- *
- * @returns Pointer to the page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    AssertGCPtr32(GCPtr);
-
-    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
-    AssertReturn(pGuestPDPT, NULL);
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-    if (pGuestPDPT->a[iPdpt].n.u1Present)
-    {
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-        PX86PDPAE   pGuestPD = NULL;
-        int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
-        AssertRCReturn(rc, NULL);
-#else
-        PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
-        if (    !pGuestPD
-            ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
-            pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
-#endif
-        return pGuestPD;
-        /* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
-    }
-    return NULL;
-}
-
-
-/**
- * Gets the page directory entry for the specified address.
- *
- * @returns Pointer to the page directory entry in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    AssertGCPtr32(GCPtr);
-
-    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
-    AssertReturn(pGuestPDPT, NULL);
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-    if (pGuestPDPT->a[iPdpt].n.u1Present)
-    {
-        const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-        PX86PDPAE   pGuestPD = NULL;
-        int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
-        AssertRCReturn(rc, NULL);
-#else
-        PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
-        if (    !pGuestPD
-            ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
-            pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
-#endif
-        return &pGuestPD->a[iPD];
-        /* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
-    }
-    return NULL;
-}
-
-
-/**
- * Gets the page directory entry for the specified address.
- *
- * @returns The page directory entry in question.
- * @returns A non-present entry if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    AssertGCPtr32(GCPtr);
-    X86PDEPAE   ZeroPde = {0};
-    PX86PDPT    pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
-    if (RT_LIKELY(pGuestPDPT))
-    {
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-        if (pGuestPDPT->a[iPdpt].n.u1Present)
-        {
-            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-            PX86PDPAE   pGuestPD = NULL;
-            int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
-            AssertRCReturn(rc, ZeroPde);
-#else
-            PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
-            if (    !pGuestPD
-                ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
-                pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
-#endif
-            return pGuestPD->a[iPD];
-        }
-    }
-    return ZeroPde;
-}
-
-
-/**
- * Gets the page directory pointer table entry for the specified address
- * and returns the index into the page directory
- *
- * @returns Pointer to the page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- * @param   piPD        Receives the index into the returned page directory
- * @param   pPdpe       Receives the page directory pointer entry. Optional.
- */
-DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
-{
-    AssertGCPtr32(GCPtr);
-
-    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
-    AssertReturn(pGuestPDPT, NULL);
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-    if (pPdpe)
-        *pPdpe = pGuestPDPT->a[iPdpt];
-    if (pGuestPDPT->a[iPdpt].n.u1Present)
-    {
-        const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-        PX86PDPAE   pGuestPD = NULL;
-        int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
-        AssertRCReturn(rc, NULL);
-#else
-        PX86PDPAE   pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
-        if (    !pGuestPD
-            ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
-            pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
-#endif
-        *piPD = iPD;
-        return pGuestPD;
-        /* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
-    }
-    return NULL;
-}
-
-#ifndef IN_RC
-
-/**
- * Gets the page map level-4 pointer for the guest.
- *
- * @returns Pointer to the PML4 page.
- * @param   pPGM        Pointer to the PGM instance data.
- */
-DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PML4 pGuestPml4;
-    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
-    AssertRCReturn(rc, NULL);
-#else
-    PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
-# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
-    if (!pGuestPml4)
-        pGuestPml4 = pgmGstLazyMapPml4(pPGM);
-# endif
-    Assert(pGuestPml4);
-#endif
-    return pGuestPml4;
-}
-
-
-/**
- * Gets the pointer to a page map level-4 entry.
- *
- * @returns Pointer to the PML4 entry.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   iPml4       The index.
- */
-DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PML4 pGuestPml4;
-    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
-    AssertRCReturn(rc, NULL);
-#else
-    PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
-# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
-    if (!pGuestPml4)
-        pGuestPml4 = pgmGstLazyMapPml4(pPGM);
-# endif
-    Assert(pGuestPml4);
-#endif
-    return &pGuestPml4->a[iPml4];
-}
-
-
-/**
- * Gets a page map level-4 entry.
- *
- * @returns The PML4 entry.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   iPml4       The index.
- */
-DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PML4 pGuestPml4;
-    int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
-    if (RT_FAILURE(rc))
-    {
-        X86PML4E ZeroPml4e = {0};
-        AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
-    }
-#else
-    PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
-# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
-    if (!pGuestPml4)
-        pGuestPml4 = pgmGstLazyMapPml4(pPGM);
-# endif
-    Assert(pGuestPml4);
-#endif
-    return pGuestPml4->a[iPml4];
-}
-
-
-/**
- * Gets the page directory pointer entry for the specified address.
- *
- * @returns Pointer to the page directory pointer entry in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- * @param   ppPml4e     Page Map Level-4 Entry (out)
- */
-DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
-{
-    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
-    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    PCX86PML4E      pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
-    if (pPml4e->n.u1Present)
-    {
-        PX86PDPT pPdpt;
-        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
-        AssertRCReturn(rc, NULL);
-
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-        return &pPdpt->a[iPdpt];
-    }
-    return NULL;
-}
-
-
-/**
- * Gets the page directory entry for the specified address.
- *
- * @returns The page directory entry in question.
- * @returns A non-present entry if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- * @param   ppPml4e     Page Map Level-4 Entry (out)
- * @param   pPdpe       Page directory pointer table entry (out)
- */
-DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
-{
-    X86PDEPAE       ZeroPde = {0};
-    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
-    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    PCX86PML4E      pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
-    if (pPml4e->n.u1Present)
-    {
-        PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
-        AssertRCReturn(rc, ZeroPde);
-
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-        *pPdpe = pPdptTemp->a[iPdpt];
-        if (pPdptTemp->a[iPdpt].n.u1Present)
-        {
-            PCX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
-            AssertRCReturn(rc, ZeroPde);
-
-            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-            return pPD->a[iPD];
-        }
-    }
-
-    return ZeroPde;
-}
-
-
-/**
- * Gets the page directory entry for the specified address.
- *
- * @returns The page directory entry in question.
- * @returns A non-present entry if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
-{
-    X86PDEPAE       ZeroPde = {0};
-    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
-    const unsigned  iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    if (pGuestPml4->a[iPml4].n.u1Present)
-    {
-        PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
-        AssertRCReturn(rc, ZeroPde);
-
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-        if (pPdptTemp->a[iPdpt].n.u1Present)
-        {
-            PCX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
-            AssertRCReturn(rc, ZeroPde);
-
-            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-            return pPD->a[iPD];
-        }
-    }
-    return ZeroPde;
-}
-
-
-/**
- * Gets the page directory entry for the specified address.
- *
- * @returns Pointer to the page directory entry in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
-{
-    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
-    const unsigned  iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    if (pGuestPml4->a[iPml4].n.u1Present)
-    {
-        PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
-        AssertRCReturn(rc, NULL);
-
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-        if (pPdptTemp->a[iPdpt].n.u1Present)
-        {
-            PX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
-            AssertRCReturn(rc, NULL);
-
-            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-            return &pPD->a[iPD];
-        }
-    }
-    return NULL;
-}
-
-
-/**
- * Gets the GUEST page directory pointer for the specified address.
- *
- * @returns The page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- * @param   ppPml4e     Page Map Level-4 Entry (out)
- * @param   pPdpe       Page directory pointer table entry (out)
- * @param   piPD        Receives the index into the returned page directory
- */
-DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe, unsigned *piPD)
-{
-    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
-    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    PCX86PML4E      pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
-    if (pPml4e->n.u1Present)
-    {
-        PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
-        AssertRCReturn(rc, NULL);
-
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-        *pPdpe = pPdptTemp->a[iPdpt];
-        if (pPdptTemp->a[iPdpt].n.u1Present)
-        {
-            PX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
-            AssertRCReturn(rc, NULL);
-
-            *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-            return pPD;
-        }
-    }
-    return 0;
-}
-
-#endif /* !IN_RC */
-
-/**
- * Gets the shadow page directory, 32-bit.
- *
- * @returns Pointer to the shadow 32-bit PD.
- * @param   pPGM        Pointer to the PGM instance data.
- */
-DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
-{
-    return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
-}
-
-
-/**
- * Gets the shadow page directory entry for the specified address, 32-bit.
- *
- * @returns Shadow 32-bit PDE.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
-
-    PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
-    if (!pShwPde)
-    {
-        X86PDE ZeroPde = {0};
-        return ZeroPde;
-    }
-    return pShwPde->a[iPd];
-}
-
-
-/**
- * Gets the pointer to the shadow page directory entry for the specified
- * address, 32-bit.
- *
- * @returns Pointer to the shadow 32-bit PDE.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
-
-    PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
-    AssertReturn(pPde, NULL);
-    return &pPde->a[iPd];
-}
-
-
-/**
- * Gets the shadow page pointer table, PAE.
- *
- * @returns Pointer to the shadow PAE PDPT.
- * @param   pPGM        Pointer to the PGM instance data.
- */
-DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
-{
-    return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
-}
-
-
-/**
- * Gets the shadow page directory for the specified address, PAE.
- *
- * @returns Pointer to the shadow PD.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-    PX86PDPT        pPdpt = pgmShwGetPaePDPTPtr(pPGM);
-
-    if (!pPdpt->a[iPdpt].n.u1Present)
-        return NULL;
-
-    /* Fetch the pgm pool shadow descriptor. */
-    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
-    AssertReturn(pShwPde, NULL);
-
-    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
-}
-
-
-/**
- * Gets the shadow page directory for the specified address, PAE.
- *
- * @returns Pointer to the shadow PD.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
-{
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-
-    if (!pPdpt->a[iPdpt].n.u1Present)
-        return NULL;
-
-    /* Fetch the pgm pool shadow descriptor. */
-    PPGMPOOLPAGE    pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
-    AssertReturn(pShwPde, NULL);
-
-    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
-}
-
-
-/**
- * Gets the shadow page directory entry, PAE.
- *
- * @returns PDE.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    const unsigned  iPd   = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-
-    PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
-    if (!pShwPde)
-    {
-        X86PDEPAE ZeroPde = {0};
-        return ZeroPde;
-    }
-    return pShwPde->a[iPd];
-}
-
-
-/**
- * Gets the pointer to the shadow page directory entry for an address, PAE.
- *
- * @returns Pointer to the PDE.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    const unsigned  iPd   = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-
-    PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
-    AssertReturn(pPde, NULL);
-    return &pPde->a[iPd];
-}
-
-#ifndef IN_RC
-
-/**
- * Gets the shadow page map level-4 pointer.
- *
- * @returns Pointer to the shadow PML4.
- * @param   pPGM        Pointer to the PGM instance data.
- */
-DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
-{
-    return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
-}
-
-
-/**
- * Gets the shadow page map level-4 entry for the specified address.
- *
- * @returns The entry.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
-{
-    const unsigned  iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    PX86PML4        pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
-
-    if (!pShwPml4)
-    {
-        X86PML4E ZeroPml4e = {0};
-        return ZeroPml4e;
-    }
-    return pShwPml4->a[iPml4];
-}
-
-
-/**
- * Gets the pointer to the specified shadow page map level-4 entry.
- *
- * @returns The entry.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   iPml4       The PML4 index.
- */
-DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
-{
-    PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
-    if (!pShwPml4)
-        return NULL;
-    return &pShwPml4->a[iPml4];
-}
-
-
-/**
- * Gets the GUEST page directory pointer for the specified address.
- *
- * @returns The page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   GCPtr       The address.
- * @param   piPD        Receives the index into the returned page directory
- */
-DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
-{
-    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
-    const unsigned  iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    if (pGuestPml4->a[iPml4].n.u1Present)
-    {
-        PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
-        AssertRCReturn(rc, NULL);
-
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-        if (pPdptTemp->a[iPdpt].n.u1Present)
-        {
-            PX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
-            AssertRCReturn(rc, NULL);
-
-            *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-            return pPD;
-        }
-    }
-    return NULL;
-}
-
-#endif /* !IN_RC */
-
-/**
- * Gets the page state for a physical handler.
- *
- * @returns The physical handler page state.
- * @param   pCur    The physical handler in question.
- */
-DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
-{
-    switch (pCur->enmType)
-    {
-        case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
-            return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
-
-        case PGMPHYSHANDLERTYPE_MMIO:
-        case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
-            return PGM_PAGE_HNDL_PHYS_STATE_ALL;
-
-        default:
-            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
-    }
-}
-
-
-/**
- * Gets the page state for a virtual handler.
- *
- * @returns The virtual handler page state.
- * @param   pCur    The virtual handler in question.
- * @remarks This should never be used on a hypervisor access handler.
- */
-DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
-{
-    switch (pCur->enmType)
-    {
-        case PGMVIRTHANDLERTYPE_WRITE:
-            return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
-        case PGMVIRTHANDLERTYPE_ALL:
-            return PGM_PAGE_HNDL_VIRT_STATE_ALL;
-        default:
-            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
-    }
-}
-
-
-/**
- * Clears one physical page of a virtual handler
- *
- * @param   pPGM    Pointer to the PGM instance.
- * @param   pCur    Virtual handler structure
- * @param   iPage   Physical page index
- *
- * @remark  Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
- *          need to care about other handlers in the same page.
- */
-DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
-{
-    const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
-
-    /*
-     * Remove the node from the tree (it's supposed to be in the tree if we get here!).
-     */
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-    AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
-                     ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                      pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
-#endif
-    if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
-    {
-        /* We're the head of the alias chain. */
-        PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-        AssertReleaseMsg(pRemove != NULL,
-                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
-        AssertReleaseMsg(pRemove == pPhys2Virt,
-                         ("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
-                          "   got:    pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
-                          pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
-#endif
-        if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
-        {
-            /* Insert the next list in the alias chain into the tree. */
-            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-            AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
-                             ("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                             pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
-#endif
-            pNext->offNextAlias |= PGMPHYS2VIRTHANDLER_IS_HEAD;
-            bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
-            AssertRelease(fRc);
-        }
-    }
-    else
-    {
-        /* Locate the previous node in the alias chain. */
-        PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-        AssertReleaseMsg(pPrev != pPhys2Virt,
-                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
-                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
-#endif
-        for (;;)
-        {
-            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
-            if (pNext == pPhys2Virt)
-            {
-                /* unlink. */
-                LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
-                         pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
-                if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
-                    pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
-                else
-                {
-                    PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
-                    pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
-                                        | (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
-                }
-                break;
-            }
-
-            /* next */
-            if (pNext == pPrev)
-            {
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-                AssertReleaseMsg(pNext != pPrev,
-                                 ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
-                                  pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
-#endif
-                break;
-            }
-            pPrev = pNext;
-        }
-    }
-    Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
-          pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
-    pPhys2Virt->offNextAlias = 0;
-    pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
-
-    /*
-     * Clear the ram flags for this page.
-     */
-    PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
-    AssertReturnVoid(pPage);
-    PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
-}
-
-
-/**
- * Internal worker for finding a 'in-use' shadow page give by it's physical address.
- *
- * @returns Pointer to the shadow page structure.
- * @param   pPool       The pool.
- * @param   idx         The pool page index.
- */
-DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
-{
-    AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
-    return &pPool->aPages[idx];
-}
-
-
-/**
- * Clear references to guest physical memory.
- *
- * @param   pPool       The pool.
- * @param   pPoolPage   The pool page.
- * @param   pPhysPage   The physical guest page tracking structure.
- */
-DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
-{
-    /*
-     * Just deal with the simple case here.
-     */
-# ifdef LOG_ENABLED
-    const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
-# endif
-    const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
-    if (cRefs == 1)
-    {
-        Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
-        PGM_PAGE_SET_TRACKING(pPhysPage, 0);
-    }
-    else
-        pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
-    Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
-}
-
-
-/**
- * Moves the page to the head of the age list.
- *
- * This is done when the cached page is used in one way or another.
- *
- * @param   pPool       The pool.
- * @param   pPage       The cached page.
- */
-DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
-{
-    PVM pVM = pPool->CTX_SUFF(pVM);
-    pgmLock(pVM);
-
-    /*
-     * Move to the head of the age list.
-     */
-    if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
-    {
-        /* unlink */
-        pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
-        if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
-            pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
-        else
-            pPool->iAgeTail = pPage->iAgePrev;
-
-        /* insert at head */
-        pPage->iAgePrev = NIL_PGMPOOL_IDX;
-        pPage->iAgeNext = pPool->iAgeHead;
-        Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
-        pPool->iAgeHead = pPage->idx;
-        pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
-    }
-    pgmUnlock(pVM);
-}
-
-/**
- * Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
- *
- * @param   pVM         VM Handle.
- * @param   pPage       PGM pool page
- */
-DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
-{
-    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
-    ASMAtomicIncU32(&pPage->cLocked);
-}
-
-
-/**
- * Unlocks a page to allow flushing again
- *
- * @param   pVM         VM Handle.
- * @param   pPage       PGM pool page
- */
-DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
-{
-    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
-    Assert(pPage->cLocked);
-    ASMAtomicDecU32(&pPage->cLocked);
-}
-
-
-/**
- * Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
- *
- * @returns VBox status code.
- * @param   pPage       PGM pool page
- */
-DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
-{
-    if (pPage->cLocked)
-    {
-        LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
-        if (pPage->cModifications)
-            pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
-        return true;
-    }
-    return false;
-}
-
-
-/**
- * Tells if mappings are to be put into the shadow page table or not.
- *
- * @returns boolean result
- * @param   pVM         VM handle.
- */
-DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
-{
-#ifdef PGM_WITHOUT_MAPPINGS
-    /* There are no mappings in VT-x and AMD-V mode. */
-    Assert(pPGM->fMappingsDisabled);
-    return false;
-#else
-    return !pPGM->fMappingsDisabled;
-#endif
-}
-
-
-/**
- * Checks if the mappings are floating and enabled.
- *
- * @returns true / false.
- * @param   pVM         The VM handle.
- */
-DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PPGM pPGM)
-{
-#ifdef PGM_WITHOUT_MAPPINGS
-    /* There are no mappings in VT-x and AMD-V mode. */
-    Assert(pPGM->fMappingsDisabled);
-    return false;
-#else
-    return !pPGM->fMappingsDisabled
-        && !pPGM->fMappingsFixed;
-#endif
-}
-
 /** @} */
 
Index: trunk/src/VBox/VMM/PGMMap.cpp
===================================================================
--- trunk/src/VBox/VMM/PGMMap.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/PGMMap.cpp	(revision 26150)
@@ -29,4 +29,5 @@
 #include "PGMInternal.h"
 #include <VBox/vm.h>
+#include "PGMInline.h"
 
 #include <VBox/log.h>
Index: trunk/src/VBox/VMM/PGMPhys.cpp
===================================================================
--- trunk/src/VBox/VMM/PGMPhys.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/PGMPhys.cpp	(revision 26150)
@@ -33,4 +33,5 @@
 #include "PGMInternal.h"
 #include <VBox/vm.h>
+#include "PGMInline.h"
 #include <VBox/sup.h>
 #include <VBox/param.h>
Index: trunk/src/VBox/VMM/PGMPool.cpp
===================================================================
--- trunk/src/VBox/VMM/PGMPool.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/PGMPool.cpp	(revision 26150)
@@ -105,4 +105,5 @@
 #include "PGMInternal.h"
 #include <VBox/vm.h>
+#include "PGMInline.h"
 
 #include <VBox/log.h>
Index: trunk/src/VBox/VMM/PGMSavedState.cpp
===================================================================
--- trunk/src/VBox/VMM/PGMSavedState.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/PGMSavedState.cpp	(revision 26150)
@@ -28,7 +28,9 @@
 #include <VBox/stam.h>
 #include <VBox/ssm.h>
-#include <VBox/pdm.h>
+#include <VBox/pdmdrv.h>
+#include <VBox/pdmdev.h>
 #include "PGMInternal.h"
 #include <VBox/vm.h>
+#include "PGMInline.h"
 
 #include <VBox/param.h>
Index: trunk/src/VBox/VMM/VMMAll/PGMAll.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMAll/PGMAll.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMAll/PGMAll.cpp	(revision 26150)
@@ -38,6 +38,7 @@
 #include <VBox/hwaccm.h>
 #include <VBox/hwacc_vmx.h>
-#include "PGMInternal.h"
+#include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
 #include <iprt/assert.h>
 #include <iprt/asm.h>
@@ -773,8 +774,8 @@
 
     /* Ignore all irrelevant error codes. */
-    if (    rc == VERR_PAGE_NOT_PRESENT                 
-        ||  rc == VERR_PAGE_TABLE_NOT_PRESENT           
-        ||  rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT   
-        ||  rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)     
+    if (    rc == VERR_PAGE_NOT_PRESENT
+        ||  rc == VERR_PAGE_TABLE_NOT_PRESENT
+        ||  rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+        ||  rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
         rc = VINF_SUCCESS;
 
Index: trunk/src/VBox/VMM/VMMAll/PGMAllHandler.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMAll/PGMAllHandler.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMAll/PGMAllHandler.cpp	(revision 26150)
@@ -34,6 +34,7 @@
 #include <VBox/dbgf.h>
 #include <VBox/rem.h>
-#include "PGMInternal.h"
+#include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
 
 #include <VBox/log.h>
Index: trunk/src/VBox/VMM/VMMAll/PGMAllMap.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMAll/PGMAllMap.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMAll/PGMAllMap.cpp	(revision 26150)
@@ -25,9 +25,10 @@
 #define LOG_GROUP LOG_GROUP_PGM
 #include <VBox/pgm.h>
-#include "PGMInternal.h"
+#include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
+#include <VBox/err.h>
+#include <iprt/asm.h>
 #include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <VBox/err.h>
 
 
Index: trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp	(revision 26150)
@@ -30,6 +30,7 @@
 #include <VBox/em.h>
 #include <VBox/rem.h>
-#include "PGMInternal.h"
+#include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
 #include <VBox/param.h>
 #include <VBox/err.h>
Index: trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp	(revision 26150)
@@ -32,6 +32,7 @@
 # include <VBox/patm.h>
 #endif
-#include "PGMInternal.h"
+#include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
 #include <VBox/disopcode.h>
 #include <VBox/hwacc_vmx.h>
Index: trunk/src/VBox/VMM/VMMGC/PGMGC.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMGC/PGMGC.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMGC/PGMGC.cpp	(revision 26150)
@@ -33,6 +33,7 @@
 #include <VBox/trpm.h>
 #include <VBox/rem.h>
-#include "PGMInternal.h"
+#include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
 
 #include <iprt/asm.h>
Index: trunk/src/VBox/VMM/VMMR0/PGMR0.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR0/PGMR0.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMR0/PGMR0.cpp	(revision 26150)
@@ -25,6 +25,7 @@
 #define LOG_GROUP LOG_GROUP_PGM
 #include <VBox/pgm.h>
-#include "PGMInternal.h"
+#include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
 #include <VBox/log.h>
 #include <VBox/err.h>
Index: trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp	(revision 26149)
+++ trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp	(revision 26150)
@@ -27,4 +27,5 @@
 #include "../PGMInternal.h"
 #include <VBox/vm.h>
+#include "../PGMInline.h"
 #include <VBox/sup.h>
 #include <VBox/err.h>
